[ Jul 19 17:44:16 Enabled. ] [ Jul 19 17:44:16 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ Jul 19 17:44:16 Method "start" exited with status 0. ] [ Jul 19 17:44:16 Rereading configuration. ] [ Jul 19 17:44:16 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:16.305886374Z","hostname":"buskin","pid":1555,"file":"illumos-utils/src/running_zone.rs:110"} {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:16.306084439Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:16.359031257Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:16.359082188Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:16.47367669Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:16.483630599Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:16.49174995Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.091825452Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fdb0:18c0:4d0d:9fb2::, width: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.231395073Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:125"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.231445115Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:137"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.231510426Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:150"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.231592669Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:172","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.231607439Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:520"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.231632919Z","hostname":"buskin","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:718"} {"msg":"Generated i86pc baseboard Pc { identifier: \"buskin\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.234784492Z","hostname":"buskin","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:625"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:17.234791972Z","hostname":"buskin","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:767"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.234808772Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:186"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.234814442Z","hostname":"buskin","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:826"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.234836583Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/m2_0.vdev"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.234904915Z","hostname":"buskin","pid":1555,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"None"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.234958636Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.235008057Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.235033678Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.246167148Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.246203199Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:372","id":"301424fa-c483-4ce9-92fb-3c8ee10e6f9f (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.344911917Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 301424fa-c483-4ce9-92fb-3c8ee10e6f9f (zpool), kind: Internal }"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.554340995Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.554378056Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/m2_1.vdev"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.554387387Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.554400757Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.554407677Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.554430478Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.554436718Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.563204476Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.563232497Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:372","id":"20dccfdd-e68a-468d-8629-7f7e0834aa1c (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.653075474Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 20dccfdd-e68a-468d-8629-7f7e0834aa1c (zpool), kind: Internal }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862436301Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_0.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862474492Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 301424fa-c483-4ce9-92fb-3c8ee10e6f9f (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 20dccfdd-e68a-468d-8629-7f7e0834aa1c (zpool), kind: Internal } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862503653Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862511093Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862519243Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862536574Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862572795Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_1.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862593015Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862598046Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 301424fa-c483-4ce9-92fb-3c8ee10e6f9f (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 20dccfdd-e68a-468d-8629-7f7e0834aa1c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862610706Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862615066Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862631346Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862634976Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_2.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862639127Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862643297Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862648677Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 301424fa-c483-4ce9-92fb-3c8ee10e6f9f (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 20dccfdd-e68a-468d-8629-7f7e0834aa1c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862664037Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_3.vdev"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862667617Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862671547Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862677057Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862681428Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862685528Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 301424fa-c483-4ce9-92fb-3c8ee10e6f9f (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 20dccfdd-e68a-468d-8629-7f7e0834aa1c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862707198Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862711649Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862715969Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_4.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862719819Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 301424fa-c483-4ce9-92fb-3c8ee10e6f9f (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 20dccfdd-e68a-468d-8629-7f7e0834aa1c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862735759Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862741309Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862745609Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862749909Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.86275413Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_5.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.86275862Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.86276223Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 301424fa-c483-4ce9-92fb-3c8ee10e6f9f (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 20dccfdd-e68a-468d-8629-7f7e0834aa1c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.86278547Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862790291Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862799261Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862802981Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_6.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862806411Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 301424fa-c483-4ce9-92fb-3c8ee10e6f9f (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 20dccfdd-e68a-468d-8629-7f7e0834aa1c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862824461Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862837472Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862841362Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862845662Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862849722Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_7.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862853802Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862858522Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862862562Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 301424fa-c483-4ce9-92fb-3c8ee10e6f9f (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 20dccfdd-e68a-468d-8629-7f7e0834aa1c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862895203Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862899703Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862903123Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_8.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862907383Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862911074Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862915024Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 301424fa-c483-4ce9-92fb-3c8ee10e6f9f (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 20dccfdd-e68a-468d-8629-7f7e0834aa1c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862941614Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:96"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862945544Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862949805Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862959865Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:98"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862966295Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862969735Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 301424fa-c483-4ce9-92fb-3c8ee10e6f9f (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 20dccfdd-e68a-468d-8629-7f7e0834aa1c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862995246Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.862999596Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.863003276Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:210"} {"msg":"No ledger in /pool/int/301424fa-c483-4ce9-92fb-3c8ee10e6f9f/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.863007466Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/20dccfdd-e68a-468d-8629-7f7e0834aa1c/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.863028047Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.863037607Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/301424fa-c483-4ce9-92fb-3c8ee10e6f9f/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.863047687Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/20dccfdd-e68a-468d-8629-7f7e0834aa1c/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.863053797Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.863060798Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.863085988Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:215"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.863090598Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:230"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.873976082Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:722"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.874016423Z","hostname":"buskin","pid":1555,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:292","time_to_next_cleanup":"599.999905848s","next_cleanup":"Instant { tv_sec: 2360, tv_nsec: 168828361 }"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.874077664Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3583"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.874085224Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"No ledger in /pool/int/301424fa-c483-4ce9-92fb-3c8ee10e6f9f/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.874138656Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/20dccfdd-e68a-468d-8629-7f7e0834aa1c/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.874148456Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.877261007Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:205"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.877332039Z","hostname":"buskin","pid":1555,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:44","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.1.164,\n last: 10.151.1.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.1.164,\n 10.151.1.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.1.100,\n vlan_id: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.1.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n}","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.877555485Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1011"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.877564565Z","hostname":"buskin","pid":1555,"component":"DnsResolver","component":"RSS","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.877608786Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"No ledger in /pool/int/301424fa-c483-4ce9-92fb-3c8ee10e6f9f/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.877650657Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/20dccfdd-e68a-468d-8629-7f7e0834aa1c/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.877659677Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.877671698Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1071"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.877697498Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"No ledger in /pool/int/301424fa-c483-4ce9-92fb-3c8ee10e6f9f/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.877707568Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/20dccfdd-e68a-468d-8629-7f7e0834aa1c/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.877715509Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.877721889Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1112"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.877725529Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:156"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:9fb2::1","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.877729059Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:129"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.87775432Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Writing ledger to /pool/int/301424fa-c483-4ce9-92fb-3c8ee10e6f9f/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.87777677Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/20dccfdd-e68a-468d-8629-7f7e0834aa1c/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.878103339Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Sled plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.878276993Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:176"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.878294984Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1145"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.878340695Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/301424fa-c483-4ce9-92fb-3c8ee10e6f9f/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.878349645Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/20dccfdd-e68a-468d-8629-7f7e0834aa1c/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.87854777Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.878728195Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.879056204Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:217","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 0c1108fb-cc91-4535-adf7-846d50ef8716, rack_id: 0ac29522-8a9f-4432-bdfe-a8adde17b3b5, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.879165966Z","hostname":"buskin","pid":1555,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:60684","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:66"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 0c1108fb-cc91-4535-adf7-846d50ef8716, rack_id: 0ac29522-8a9f-4432-bdfe-a8adde17b3b5, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.87929127Z","hostname":"buskin","pid":1555,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:360"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.87929907Z","hostname":"buskin","pid":1555,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:375"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.879337781Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/server.rs:44"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.879356622Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.879360322Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:572"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.879363522Z","hostname":"buskin","pid":1555,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.879368032Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:101::, width: 64 }"} {"msg":"No ledger in /pool/int/301424fa-c483-4ce9-92fb-3c8ee10e6f9f/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.879390112Z","hostname":"buskin","pid":1555,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/20dccfdd-e68a-468d-8629-7f7e0834aa1c/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.879402523Z","hostname":"buskin","pid":1555,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.879407823Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:561"} {"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.879411063Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:589"} {"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.879415803Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:393"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.879443234Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:374"} {"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.879448214Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:479","result":"DisksManagementResult { status: [] }"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.8815922Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3815"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.913951542Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.934533867Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.952528045Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.997529066Z","hostname":"buskin","pid":1555,"sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:366"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.997609848Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.997618578Z","hostname":"buskin","pid":1555,"sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:378"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:17.997630968Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 50.840126ms","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:18.000525644Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"68a98f37-3ce8-479e-82e7-f6f6c6391517\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 17:44:17 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"68a98f37-3ce8-479e-82e7-f6f6c6391517\" }"} {"msg":"attempting to create encrypted zvol: name=\"oxi_301424fa-c483-4ce9-92fb-3c8ee10e6f9f/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:18.010614236Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"successfully created encrypted zvol: name=\"oxi_301424fa-c483-4ce9-92fb-3c8ee10e6f9f/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:18.035301788Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_301424fa-c483-4ce9-92fb-3c8ee10e6f9f/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:18.03534492Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:18.040989967Z","hostname":"buskin","pid":1555,"sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:393"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:18.041003717Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:126"} {"msg":"Ensuring dataset oxi_301424fa-c483-4ce9-92fb-3c8ee10e6f9f/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:18.041009057Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:136"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:18.053453491Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 72.232124ms","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:18.053996255Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a652dc9c-2a64-4f99-b5c3-7a5b6f0aa923\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 17:44:17 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a652dc9c-2a64-4f99-b5c3-7a5b6f0aa923\" }"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:18.115814863Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:170"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:18.127869047Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 137.365348ms","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:18.128426751Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7db990ec-e77b-48bc-a8d9-f3174b000959\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 17:44:18 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7db990ec-e77b-48bc-a8d9-f3174b000959\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:18.266719339Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 120.102669ms","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:18.267341925Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"35c0ef17-3dc8-45b9-8a3c-0b74aee0603b\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 17:44:18 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"35c0ef17-3dc8-45b9-8a3c-0b74aee0603b\" }"} {"msg":"Mounting oxi_301424fa-c483-4ce9-92fb-3c8ee10e6f9f/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:18.334279247Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:178"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:18.347216564Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:18.347306146Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:18.347338357Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:18.347373968Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:192"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:18.388424416Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 104.992291ms","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:18.388953529Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"535f9f29-935e-44c3-b529-ff7eada01c20\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 17:44:18 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"535f9f29-935e-44c3-b529-ff7eada01c20\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:18.494812554Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 157.48605ms","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:18.495515102Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"08626959-1068-4cfb-8914-aec32e8af99d\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 17:44:18 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"08626959-1068-4cfb-8914-aec32e8af99d\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:18.653985435Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 378.194517ms","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:18.654683073Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f9741c56-15e2-494f-a276-b3cb428dab8a\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 17:44:18 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f9741c56-15e2-494f-a276-b3cb428dab8a\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:19.033479218Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:19.073319254Z","hostname":"buskin","pid":1555,"sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:90"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:19.0739203Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:56984","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:205","component":"producer-server","component":"MetricsManager","sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:205"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:19.073979272Z","hostname":"buskin","pid":1555,"component":"internal-dns-resolver","file":"internal-dns/src/resolver.rs:60","component":"producer-server","component":"MetricsManager","sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:19.074024843Z","hostname":"buskin","pid":1555,"file":"oximeter/producer/src/lib.rs:283","component":"producer-server","component":"MetricsManager","sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"oximeter/producer/src/lib.rs:283","interval":"30s","address":"[fd00:1122:3344:101::1]:56984","producer_id":"0c1108fb-cc91-4535-adf7-846d50ef8716"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:19.075175553Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","component":"MetricsManager","sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"203.361139ms"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:19.119358012Z","hostname":"buskin","pid":1555,"component":"VmmReservoirManager","sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:19.119392783Z","hostname":"buskin","pid":1555,"component":"VmmReservoirManager","sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:19.119432964Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:905","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:19.119444225Z","hostname":"buskin","pid":1555,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:19.120135303Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:205"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:19.120164673Z","hostname":"buskin","pid":1555,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:430"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:19.120192084Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Writing ledger to /pool/int/301424fa-c483-4ce9-92fb-3c8ee10e6f9f/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:19.120209854Z","hostname":"buskin","pid":1555,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/20dccfdd-e68a-468d-8629-7f7e0834aa1c/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:19.120464341Z","hostname":"buskin","pid":1555,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:19.120649206Z","hostname":"buskin","pid":1555,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:19.120668986Z","hostname":"buskin","pid":1555,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV2 { rack_subnet: Ipv6Net { addr: fd00:1122:3344:100::, width: 56 }, infra_ip_first: 10.151.1.174, infra_ip_last: 10.151.1.174, ports: [PortConfigV2 { routes: [RouteConfig { destination: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), nexthop: 10.151.1.100, vlan_id: None }], addresses: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false }], bgp: [], bfd: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:19.120681367Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3583"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:19.120712978Z","hostname":"buskin","pid":1555,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:60684","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:73"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:19.318731429Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:108","peer":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:19.318807421Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:19.319052038Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"No ledger in /pool/int/301424fa-c483-4ce9-92fb-3c8ee10e6f9f/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:19.319181891Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/20dccfdd-e68a-468d-8629-7f7e0834aa1c/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:19.319206562Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:19.319251843Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:19.319283154Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:19.320251569Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","component":"MetricsManager","sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"732.114598ms"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/0c1108fb-cc91-4535-adf7-846d50ef8716): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:19.325233319Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:19.351378949Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:64204"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:19.351555383Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:19.351644036Z","hostname":"buskin","pid":1555,"uri":"/inventory","method":"GET","req_id":"01e73c61-a89c-4d5d-bfd4-1513a9fc6a3d","remote_addr":"[fd00:1122:3344:101::1]:64204","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":169,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:19.38063625Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:57380"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:19.380715942Z","hostname":"buskin","pid":1555,"uri":"/sled-role","method":"GET","req_id":"9bbb54d6-e162-4d79-b430-5b004da1e125","remote_addr":"[fd00:1122:3344:101::1]:57380","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":43,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:19.381363739Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Writing ledger to /pool/int/301424fa-c483-4ce9-92fb-3c8ee10e6f9f/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:19.38138826Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/20dccfdd-e68a-468d-8629-7f7e0834aa1c/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:19.381712558Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:19.382049627Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:832"} {"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { disks: [OmicronPhysicalDiskConfig { id: b569542b-5f11-4b90-84d5-387811c14788, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, pool_id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool) }, OmicronPhysicalDiskConfig { id: b06cbc3e-b215-4c8e-aed8-166fadbae657, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, pool_id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool) }, OmicronPhysicalDiskConfig { id: 2978d5b5-aa61-4523-889e-0407173d3a4e, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, pool_id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool) }, OmicronPhysicalDiskConfig { id: 593121ba-1210-4f7a-b04a-8c7ecd0ef1f9, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, pool_id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool) }, OmicronPhysicalDiskConfig { id: 5939261f-cd52-4755-bd75-57dec66dea03, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, pool_id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool) }, OmicronPhysicalDiskConfig { id: 039410aa-de10-4420-b346-3db59201d69b, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, pool_id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool) }, OmicronPhysicalDiskConfig { id: ebc8c4cf-db7e-4645-b934-415332d8354e, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, pool_id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool) }, OmicronPhysicalDiskConfig { id: 76c9fa56-82f8-4131-a2ba-1b9a8c208bf2, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, pool_id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool) }, OmicronPhysicalDiskConfig { id: 6eb91952-c334-44c1-807e-728f7748b534, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, pool_id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool) }], generation: Generation(1) }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:19.410743093Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:348"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:19.410838536Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:49963"} {"msg":"physical disks ensure","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:19.411136234Z","hostname":"buskin","pid":1555,"sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:825"} {"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: b569542b-5f11-4b90-84d5-387811c14788, pool_id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: b06cbc3e-b215-4c8e-aed8-166fadbae657, pool_id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: 2978d5b5-aa61-4523-889e-0407173d3a4e, pool_id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 593121ba-1210-4f7a-b04a-8c7ecd0ef1f9, pool_id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: 5939261f-cd52-4755-bd75-57dec66dea03, pool_id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: 039410aa-de10-4420-b346-3db59201d69b, pool_id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: ebc8c4cf-db7e-4645-b934-415332d8354e, pool_id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 76c9fa56-82f8-4131-a2ba-1b9a8c208bf2, pool_id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: 6eb91952-c334-44c1-807e-728f7748b534, pool_id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:19.411186615Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"No ledger in /pool/int/301424fa-c483-4ce9-92fb-3c8ee10e6f9f/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:19.411191695Z","hostname":"buskin","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/20dccfdd-e68a-468d-8629-7f7e0834aa1c/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:19.411207885Z","hostname":"buskin","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:19.411217786Z","hostname":"buskin","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:643"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:19.411268327Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:374"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:19.411273327Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:19.411278417Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:19.411287227Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:19.533055126Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:19.533102997Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool)"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:20.191236178Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","component":"MetricsManager","sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"785.593685ms"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/0c1108fb-cc91-4535-adf7-846d50ef8716): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-19T17:44:20.191342591Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:20.611654067Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:20.808296193Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:20.808390945Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:20.808397515Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:20.808403675Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:20.808409075Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:20.808541519Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:20.808547049Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:21.030568735Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","component":"MetricsManager","sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"1.415004669s"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:21.230199579Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/0c1108fb-cc91-4535-adf7-846d50ef8716): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-19T17:44:21.921170736Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/0c1108fb-cc91-4535-adf7-846d50ef8716): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:21.921223927Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:21.921377162Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 50.769637ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:22.137960536Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Automatically destroying dataset: oxp_988e775a-ddc4-455d-bc4e-f2e3c228cc74/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:22.151587151Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 125.633507ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:22.755316248Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:22.756967611Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","component":"MetricsManager","sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"5.870274815s"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/0c1108fb-cc91-4535-adf7-846d50ef8716): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-19T17:44:23.20587455Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 202.191772ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:23.39768504Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 421.610628ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:24.000500824Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/0c1108fb-cc91-4535-adf7-846d50ef8716): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:24.002389373Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/0c1108fb-cc91-4535-adf7-846d50ef8716): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-19T17:44:24.207716055Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 803.068703ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:25.075259146Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Looking for unencrypted datasets in oxp_988e775a-ddc4-455d-bc4e-f2e3c228cc74","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:25.088647794Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/0c1108fb-cc91-4535-adf7-846d50ef8716): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-19T17:44:25.284078459Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:25.290118496Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:25.290129596Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:25.290142137Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:25.290147757Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:25.290164917Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:25.506322451Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:25.506357232Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:26.582463779Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:26.804438714Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:26.804484165Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:26.804490086Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:26.804525106Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:26.804533307Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:26.80467042Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:26.80467647Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/0c1108fb-cc91-4535-adf7-846d50ef8716): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:27.464321472Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/0c1108fb-cc91-4535-adf7-846d50ef8716): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-19T17:44:27.464375873Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:28.132106035Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_f3be927c-0009-45ce-8f85-0c94f9b67b8f/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:28.602115473Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:29.081962277Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","component":"MetricsManager","sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"7.012443366s"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/0c1108fb-cc91-4535-adf7-846d50ef8716): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-19T17:44:29.3096512Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/0c1108fb-cc91-4535-adf7-846d50ef8716): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:29.309700922Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:29.51603736Z","hostname":"buskin","pid":1555,"component":"VmmReservoirManager","sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"} {"msg":"Looking for unencrypted datasets in oxp_f3be927c-0009-45ce-8f85-0c94f9b67b8f","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:29.623990448Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:29.63211673Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:29.632156841Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:29.632165461Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:29.632171621Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:29.632176821Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:29.641910435Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:29.641939066Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:29.735227443Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","zpool":"ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:29.74320586Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:29.743240581Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:29.743251241Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:29.743295412Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:29.743308993Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:29.743448547Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:29.743456497Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:29.779847903Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:29.787595665Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:29.910335178Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:29.918145472Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:29.918176093Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:29.918194173Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:29.918201123Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:29.918205763Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:29.927911076Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:29.927938416Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.020913585Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","zpool":"ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.029602531Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.029626762Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.029633182Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.029649173Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.029656233Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.029776986Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.029782616Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.066339267Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_4c50992d-c53d-42b7-a46f-5340e3aee556/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.075045924Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_4c50992d-c53d-42b7-a46f-5340e3aee556","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.200556759Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.208572097Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.208603848Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.208613759Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.208619729Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.208628759Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.218568118Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.218611569Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.310611212Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","zpool":"ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.318387585Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.318424606Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.318443736Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.318497048Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.318509258Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.318644421Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.318651432Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/0c1108fb-cc91-4535-adf7-846d50ef8716): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-19T17:44:30.323145158Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.354806342Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_7c02949e-12b1-4d82-9ac3-2efd807aef95/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.362282297Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_7c02949e-12b1-4d82-9ac3-2efd807aef95","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.484200369Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.491993111Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.492030722Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.492036583Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.492059573Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.492064553Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.501630862Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.501648302Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.593101142Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","zpool":"ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.600002292Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.600038162Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.600044542Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.600064213Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.600074763Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.600240308Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.600245788Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.632720263Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_765bc25e-72d0-4d30-8cfe-a58b7db5c2e8/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.640213558Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_765bc25e-72d0-4d30-8cfe-a58b7db5c2e8","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.758339921Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.765335203Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.765356574Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.765362424Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.765367744Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.765371664Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.773967598Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.773984678Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"696d009b-6d0c-4082-b81b-231c6c85d568 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.86937432Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","zpool":"ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.877100651Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.877128051Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.877162932Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.877198523Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.877205993Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.877331657Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.877340507Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/0c1108fb-cc91-4535-adf7-846d50ef8716): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:30.880643693Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Ensuring tfportrear0_0/ll exists in zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.908239911Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2256"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.918073577Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_696d009b-6d0c-4082-b81b-231c6c85d568/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:30.925863879Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_696d009b-6d0c-4082-b81b-231c6c85d568","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.054716762Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.062500144Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.062532005Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.062539275Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.062544145Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.062558596Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.072767871Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.072811893Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.167302241Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","zpool":"ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.175847913Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.175891814Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.175902855Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.175917255Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.175928355Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.176065919Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.176074769Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.213188905Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_311acf6c-db4a-40cb-bd1f-e33d2bc8747f/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.220973957Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/0c1108fb-cc91-4535-adf7-846d50ef8716): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-19T17:44:31.324906251Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Looking for unencrypted datasets in oxp_311acf6c-db4a-40cb-bd1f-e33d2bc8747f","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.341450242Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.349247885Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.349280845Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.349288075Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.349306556Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.349323356Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.359106301Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.359138922Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"41faf794-cdf9-49a0-b529-4c18e335c28c (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.449805081Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","zpool":"ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.456824963Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.456857454Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.456870414Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.456887775Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.456894545Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.457038479Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.457044429Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.490308724Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_41faf794-cdf9-49a0-b529-4c18e335c28c/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.498352054Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_41faf794-cdf9-49a0-b529-4c18e335c28c","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.62464815Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.633580382Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.633642784Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Writing ledger to /pool/int/301424fa-c483-4ce9-92fb-3c8ee10e6f9f/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.633656944Z","hostname":"buskin","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.633801748Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 301424fa-c483-4ce9-92fb-3c8ee10e6f9f (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 20dccfdd-e68a-468d-8629-7f7e0834aa1c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.63388588Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.6338975Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Writing ledger to /pool/int/20dccfdd-e68a-468d-8629-7f7e0834aa1c/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.633939421Z","hostname":"buskin","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"physical disks ensure: Updated storage","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.634143967Z","hostname":"buskin","pid":1555,"sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:829"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.634163547Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"physical disks ensure: Propagating new generation of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.634169867Z","hostname":"buskin","pid":1555,"sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:855","generation":"Generation(1)"} {"msg":"physical disks ensure: Updated storage monitor","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.634176558Z","hostname":"buskin","pid":1555,"sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:860"} {"msg":"physical disks ensure: Updated zone bundler","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.634196298Z","hostname":"buskin","pid":1555,"sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:865"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.634202888Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:116","gen":"Generation(1)"} {"msg":"physical disks ensure: Updated probes","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.634211728Z","hostname":"buskin","pid":1555,"sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:871"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.63426428Z","hostname":"buskin","pid":1555,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:822","gen":"Generation(1)"} {"msg":"physical disks ensure: Updated instances","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.63427366Z","hostname":"buskin","pid":1555,"sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:876"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.634355602Z","hostname":"buskin","pid":1555,"uri":"/omicron-physical-disks","method":"PUT","req_id":"5719d743-a8d4-4691-861c-93baccb007f0","remote_addr":"[fd00:1122:3344:101::1]:49963","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":12223353,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: 8ae14f97-bd01-42ad-87d5-23eae9194521, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.66504131Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.665181834Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:38265"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.665466602Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.665473742Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.716620423Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.716647743Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.766977893Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.767003193Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.817521298Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"No ledger in /pool/int/301424fa-c483-4ce9-92fb-3c8ee10e6f9f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.817553609Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/20dccfdd-e68a-468d-8629-7f7e0834aa1c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.817564609Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.8176194Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.862414766Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.862446106Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.862451077Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_98b939a3-2514-4766-b854-3b3ddf8b23f9","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.914722717Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_8f9c564d-5823-4b85-a0ff-2de7f54b602e","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.918626128Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_8ae14f97-bd01-42ad-87d5-23eae9194521","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.922676464Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_internal_dns_98b939a3-2514-4766-b854-3b3ddf8b23f9","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.934018528Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_8f9c564d-5823-4b85-a0ff-2de7f54b602e","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.93831453Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_8ae14f97-bd01-42ad-87d5-23eae9194521","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:31.942824408Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/0c1108fb-cc91-4535-adf7-846d50ef8716): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-19T17:44:32.326876809Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/0c1108fb-cc91-4535-adf7-846d50ef8716): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:32.879342363Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Profile for oxz_internal_dns_8ae14f97-bd01-42ad-87d5-23eae9194521:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:33.456448027Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:33.456522169Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:1::, width: 64 }"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:33.456723884Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_8ae14f97-bd01-42ad-87d5-23eae9194521","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Profile for oxz_internal_dns_98b939a3-2514-4766-b854-3b3ddf8b23f9:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:33.943195501Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:34.422342626Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:2::, width: 64 }"} {"msg":"Profile for oxz_internal_dns_8f9c564d-5823-4b85-a0ff-2de7f54b602e:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:34.422422888Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_internal_dns_8ae14f97-bd01-42ad-87d5-23eae9194521\") failed: Property not found. retry in 51.214154ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:34.429282317Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_8ae14f97-bd01-42ad-87d5-23eae9194521","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:34.429310438Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:3::, width: 64 }"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:34.429365179Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_98b939a3-2514-4766-b854-3b3ddf8b23f9","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:34.4332377Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_8f9c564d-5823-4b85-a0ff-2de7f54b602e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"wait for service Some(\"oxz_internal_dns_8ae14f97-bd01-42ad-87d5-23eae9194521\") failed: Property not found. retry in 62.81744ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:34.489656377Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_8ae14f97-bd01-42ad-87d5-23eae9194521","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_98b939a3-2514-4766-b854-3b3ddf8b23f9\") failed: Property not found. retry in 49.494093ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:34.556274301Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_98b939a3-2514-4766-b854-3b3ddf8b23f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_8f9c564d-5823-4b85-a0ff-2de7f54b602e\") failed: Property not found. retry in 39.593444ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:34.562483712Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_8f9c564d-5823-4b85-a0ff-2de7f54b602e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_8ae14f97-bd01-42ad-87d5-23eae9194521\") failed: Property not found. retry in 117.421629ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:34.569278789Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_8ae14f97-bd01-42ad-87d5-23eae9194521","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_8f9c564d-5823-4b85-a0ff-2de7f54b602e\") failed: Property not found. retry in 53.914648ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:34.608561221Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_8f9c564d-5823-4b85-a0ff-2de7f54b602e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_98b939a3-2514-4766-b854-3b3ddf8b23f9\") failed: Property not found. retry in 122.57237ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:34.614710071Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_98b939a3-2514-4766-b854-3b3ddf8b23f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_8f9c564d-5823-4b85-a0ff-2de7f54b602e\") failed: Property not found. retry in 235.381217ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:34.707091525Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_8f9c564d-5823-4b85-a0ff-2de7f54b602e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_8ae14f97-bd01-42ad-87d5-23eae9194521\") failed: Property not found. retry in 410.878053ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:34.729478177Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_8ae14f97-bd01-42ad-87d5-23eae9194521","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_98b939a3-2514-4766-b854-3b3ddf8b23f9\") failed: Property not found. retry in 117.60975ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:34.751270514Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_98b939a3-2514-4766-b854-3b3ddf8b23f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_98b939a3-2514-4766-b854-3b3ddf8b23f9\") failed: Property not found. retry in 461.270286ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:34.896699108Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_98b939a3-2514-4766-b854-3b3ddf8b23f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_8f9c564d-5823-4b85-a0ff-2de7f54b602e\") failed: Property not found. retry in 224.149966ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:34.960316833Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_8f9c564d-5823-4b85-a0ff-2de7f54b602e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_8ae14f97-bd01-42ad-87d5-23eae9194521\") failed: Property not found. retry in 576.269057ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:35.150828909Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_8ae14f97-bd01-42ad-87d5-23eae9194521","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_8f9c564d-5823-4b85-a0ff-2de7f54b602e\") failed: Property not found. retry in 726.229593ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:35.195419429Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_8f9c564d-5823-4b85-a0ff-2de7f54b602e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_98b939a3-2514-4766-b854-3b3ddf8b23f9\") failed: Property not found. retry in 717.225762ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:35.366779518Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_98b939a3-2514-4766-b854-3b3ddf8b23f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_8ae14f97-bd01-42ad-87d5-23eae9194521\") failed: Property not found. retry in 1.24587099s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:35.736656601Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_8ae14f97-bd01-42ad-87d5-23eae9194521","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_8f9c564d-5823-4b85-a0ff-2de7f54b602e\") failed: Property not found. retry in 1.45186029s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:35.932565188Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_8f9c564d-5823-4b85-a0ff-2de7f54b602e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_98b939a3-2514-4766-b854-3b3ddf8b23f9\") failed: Property not found. retry in 539.821822ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:36.093518235Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_98b939a3-2514-4766-b854-3b3ddf8b23f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_98b939a3-2514-4766-b854-3b3ddf8b23f9\") failed: Property not found. retry in 936.663938ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:36.641938373Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_98b939a3-2514-4766-b854-3b3ddf8b23f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring bootstrap address fdb0:18c0:4d0d:9fb2::2 exists in switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:36.963319395Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2273"} {"msg":"Adding bootstrap address","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:36.963338875Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:578"} {"msg":"wait for service Some(\"oxz_internal_dns_8ae14f97-bd01-42ad-87d5-23eae9194521\") failed: Property not found. retry in 1.335452263s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:36.990898452Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_8ae14f97-bd01-42ad-87d5-23eae9194521","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_8f9c564d-5823-4b85-a0ff-2de7f54b602e\") failed: Property not found. retry in 665.627697ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:37.392799828Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_8f9c564d-5823-4b85-a0ff-2de7f54b602e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Forwarding bootstrap traffic via oxBootstrap0 to fe80::8:20ff:fea8:ad0b","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:37.507644956Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2280"} {"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:37.51279262Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2341"} {"msg":"wait for service Some(\"oxz_internal_dns_98b939a3-2514-4766-b854-3b3ddf8b23f9\") failed: Property not found. retry in 1.46205302s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:37.587543325Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_98b939a3-2514-4766-b854-3b3ddf8b23f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:37.635261867Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2526"} {"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:37.876995566Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2690"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:38.062691877Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2405"} {"msg":"wait for service Some(\"oxz_internal_dns_8f9c564d-5823-4b85-a0ff-2de7f54b602e\") failed: Property not found. retry in 883.095783ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:38.067080161Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_8f9c564d-5823-4b85-a0ff-2de7f54b602e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_8ae14f97-bd01-42ad-87d5-23eae9194521\") failed: Property not found. retry in 1.111038184s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:38.335441373Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_8ae14f97-bd01-42ad-87d5-23eae9194521","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:38.364602342Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2440"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:38.583960189Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2740"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:38.770626715Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2767"} {"msg":"wait for service Some(\"oxz_internal_dns_8f9c564d-5823-4b85-a0ff-2de7f54b602e\") failed: Property not found. retry in 574.607279ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:38.960549727Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_8f9c564d-5823-4b85-a0ff-2de7f54b602e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:38.987525237Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2634"} {"msg":"wait for service Some(\"oxz_internal_dns_98b939a3-2514-4766-b854-3b3ddf8b23f9\") failed: Property not found. retry in 1.486338133s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:39.059957782Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_98b939a3-2514-4766-b854-3b3ddf8b23f9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:39.173518296Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2434"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:39.183750513Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3839","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:39.183790714Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3855"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:39.183801004Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:561"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:39.741822802Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3863"} {"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:39.74865854Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3919"} {"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:39.892845401Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3981"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:39.991511048Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3962","rack_subnet":"fd00:1122:3344::"} {"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.016262202Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4009"} {"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.109952519Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4036"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.228556165Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:369"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.260800804Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:417"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.293140465Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:656"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.293222248Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:694"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.293660789Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:697","uptime":"3"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.293669909Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:439","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.1.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n}"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.370594111Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.370688903Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.370704323Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.370717824Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.370729984Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.370739555Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.370751275Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.370760705Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.370785756Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.370798166Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.370804406Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.370810796Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.370814937Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.370818906Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.370823017Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.370827017Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.370831037Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.370834987Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.370838867Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.370842907Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.370846857Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.370860338Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.370864608Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.566143308Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_internal_dns_98b939a3-2514-4766-b854-3b3ddf8b23f9"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.566188729Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_internal_dns_8f9c564d-5823-4b85-a0ff-2de7f54b602e"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.56619592Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_internal_dns_8ae14f97-bd01-42ad-87d5-23eae9194521"} {"msg":"Writing ledger to /pool/int/301424fa-c483-4ce9-92fb-3c8ee10e6f9f/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.56620609Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/20dccfdd-e68a-468d-8629-7f7e0834aa1c/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.566541189Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.566832636Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"e36d444e-2d9d-4629-bb6e-d299c450aefa","remote_addr":"[fd00:1122:3344:101::1]:38265","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":8901562,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.568466849Z","hostname":"buskin","pid":1555,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:549"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.619543298Z","hostname":"buskin","pid":1555,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:549"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.666718685Z","hostname":"buskin","pid":1555,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:549"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.711808868Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:585"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.711818028Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:263"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.743420031Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.744246672Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:305"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.744255302Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: 8ae14f97-bd01-42ad-87d5-23eae9194521, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: bc60810c-eece-4cbd-a0f8-d04271924b7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bf3fc707-b452-4d30-a92b-e56b4d4122ee, kind: Service { id: bc60810c-eece-4cbd-a0f8-d04271924b7c }, name: Name(\"ntp-bc60810c-eece-4cbd-a0f8-d04271924b7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 196])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.772820656Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.772925238Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:47207"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.773665018Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.773691398Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.794827968Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.794871339Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.812437496Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.812479317Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.830010463Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/301424fa-c483-4ce9-92fb-3c8ee10e6f9f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.830068145Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/20dccfdd-e68a-468d-8629-7f7e0834aa1c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.830226689Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.830333082Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.841646926Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:263"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.873788702Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.874396558Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:305"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.874409499Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.903514196Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:426","port":"Port { inner: PortInner(PortData { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 128, 196]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.903526346Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1210","zone_type":"ntp"} {"msg":"no nat entry found for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.904532052Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.904538823Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.904781389Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.90598391Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Configuring new Omicron zone: oxz_ntp_bc60810c-eece-4cbd-a0f8-d04271924b7c","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.928426714Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_ntp_bc60810c-eece-4cbd-a0f8-d04271924b7c","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:40.946898785Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_ntp_bc60810c-eece-4cbd-a0f8-d04271924b7c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:41.564605244Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:41.564660856Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_bc60810c-eece-4cbd-a0f8-d04271924b7c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"wait for service Some(\"oxz_ntp_bc60810c-eece-4cbd-a0f8-d04271924b7c\") failed: Property not found. retry in 62.163952ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:41.704250228Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_bc60810c-eece-4cbd-a0f8-d04271924b7c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_bc60810c-eece-4cbd-a0f8-d04271924b7c\") failed: Property not found. retry in 61.933489ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:41.784693421Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_bc60810c-eece-4cbd-a0f8-d04271924b7c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_bc60810c-eece-4cbd-a0f8-d04271924b7c\") failed: Property not found. retry in 119.907276ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:41.875436291Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_bc60810c-eece-4cbd-a0f8-d04271924b7c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_bc60810c-eece-4cbd-a0f8-d04271924b7c\") failed: Property not found. retry in 419.736462ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:42.006898802Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_bc60810c-eece-4cbd-a0f8-d04271924b7c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_bc60810c-eece-4cbd-a0f8-d04271924b7c\") failed: Property not found. retry in 452.468173ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:42.436120249Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_bc60810c-eece-4cbd-a0f8-d04271924b7c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_bc60810c-eece-4cbd-a0f8-d04271924b7c\") failed: Property not found. retry in 1.343545497s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:42.898387556Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_bc60810c-eece-4cbd-a0f8-d04271924b7c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:43.660183025Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:39758"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:43.66038674Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"ec2664e0-3bea-4525-93e1-a55f56e8dae1","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:39758","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":94,"response_code":"200"} {"msg":"wait for service Some(\"oxz_ntp_bc60810c-eece-4cbd-a0f8-d04271924b7c\") failed: Property not found. retry in 1.268702518s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:44.25118418Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_bc60810c-eece-4cbd-a0f8-d04271924b7c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_bc60810c-eece-4cbd-a0f8-d04271924b7c\") failed: Property not found. retry in 958.922092ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:45.530824343Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_bc60810c-eece-4cbd-a0f8-d04271924b7c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:46.511683032Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_ntp_bc60810c-eece-4cbd-a0f8-d04271924b7c"} {"msg":"Writing ledger to /pool/int/301424fa-c483-4ce9-92fb-3c8ee10e6f9f/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:46.511738593Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/20dccfdd-e68a-468d-8629-7f7e0834aa1c/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:46.512036631Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:46.512338089Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"c59a0d6d-0d55-4a8b-9f52-91f0e2c6b509","remote_addr":"[fd00:1122:3344:101::1]:47207","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":5739335,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:46.514490105Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:626"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:46.544189357Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:46.544394083Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:45764"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_bc60810c-eece-4cbd-a0f8-d04271924b7c': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:46.555003629Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3569"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:46.555121652Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"e0d506fb-a643-4885-8e95-1a6d48c559e9","remote_addr":"[fd00:1122:3344:101::1]:45764","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":10651,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:46.55851251Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:46.651184031Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:46.651319395Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:53542"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_bc60810c-eece-4cbd-a0f8-d04271924b7c': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:50.660053349Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3569"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:50.660164872Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"5396127a-d5ac-447d-9b0f-046bba1c466e","remote_addr":"[fd00:1122:3344:101::1]:53542","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4008713,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:50.66357441Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:50.835832122Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:50.835960655Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:42248"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_bc60810c-eece-4cbd-a0f8-d04271924b7c': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:50.844491027Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3569"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:50.844630031Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"51dfcd2e-7c40-43a5-a9bf-10b53638892d","remote_addr":"[fd00:1122:3344:101::1]:42248","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8551,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:50.84766526Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:51.085430955Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:51.085545868Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:38576"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_bc60810c-eece-4cbd-a0f8-d04271924b7c': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:51.093961077Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3569"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:51.09409708Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"52c7e2a7-bd64-4c83-887e-4e1026b34aa3","remote_addr":"[fd00:1122:3344:101::1]:38576","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8448,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:51.095835535Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:51.236641079Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:51.236760772Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:41265"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:55.245708551Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"f28f0740-d8a1-491b-8d73-cb1161949a90","remote_addr":"[fd00:1122:3344:101::1]:41265","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":4008797,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:55.248387241Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:55.248399221Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:55.55086694Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:55.551006334Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:56295"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:55.56008176Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"f1f956ce-bf69-49e3-83b9-d13aa1d942b7","remote_addr":"[fd00:1122:3344:101::1]:56295","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":8966,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:55.563364685Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:55.563374716Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:55.891117883Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:55.891225225Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:35628"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:55.899841229Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"95b8b527-1052-42f1-b80e-e81ec633122e","remote_addr":"[fd00:1122:3344:101::1]:35628","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":8535,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:55.901569614Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:55.901575824Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:56.1031998Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:56.103312893Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:42472"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:56.11165395Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"fe78d930-fa06-4b87-96b0-0493d69d3c91","remote_addr":"[fd00:1122:3344:101::1]:42472","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":8233,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:56.115262804Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:56.115278324Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:56.134754541Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"130.376962ms"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:56.661014713Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:56.661119616Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:33129"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:56.670103019Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"312d7ad3-5845-4717-818c-d2efc10f3c38","remote_addr":"[fd00:1122:3344:101::1]:33129","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":8843,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:56.673033595Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:56.673044196Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:56.923181114Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:56.923294217Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:35152"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:56.931291475Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"a56ffd73-4675-4fe2-b3a5-d0c47ab3af2b","remote_addr":"[fd00:1122:3344:101::1]:35152","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":7839,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:56.933449541Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:56.933459731Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:57.364710541Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:57.364810564Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:34493"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:57.372998397Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"c2604e9c-fd1b-407f-ab9d-b7f884d00c0f","remote_addr":"[fd00:1122:3344:101::1]:34493","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":8046,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:57.375010999Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:44:57.375016669Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:58.37207767Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:58.372189083Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:53905"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:58.392886042Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"15aa1b5f-a76b-4d61-b2f0-3b950683217c","remote_addr":"[fd00:1122:3344:101::1]:53905","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":20567,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 1241950281, ip_addr: 74.6.168.73, stratum: 3, ref_time: 1721411097.3841624, correction: -0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:58.395716285Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:58.395732705Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1224"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 8ae14f97-bd01-42ad-87d5-23eae9194521, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: bc60810c-eece-4cbd-a0f8-d04271924b7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bf3fc707-b452-4d30-a92b-e56b4d4122ee, kind: Service { id: bc60810c-eece-4cbd-a0f8-d04271924b7c }, name: Name(\"ntp-bc60810c-eece-4cbd-a0f8-d04271924b7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 196])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: c168b27b-f467-4350-a348-4006ce85a213, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 15be01e5-1765-45f0-b521-da99b395bdb3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: d8a1310d-09c1-4844-91b7-509f93b30cfd, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:58.42551139Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:58.425634243Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:54642"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:58.425943791Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:58.425954262Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:58.445460759Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:58.44550672Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:58.462565244Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:58.462602035Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:58.480878121Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:58.480926792Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:58.532642718Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:58.532692659Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:58.583707446Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:58.583746817Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:58.63573688Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:58.635772921Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:58.687447395Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:58.687487576Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:58.740492285Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/301424fa-c483-4ce9-92fb-3c8ee10e6f9f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:58.740547196Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/20dccfdd-e68a-468d-8629-7f7e0834aa1c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:58.740811034Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:58.740937387Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:58.825318382Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:58.825374654Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:58.825381804Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:58.825389814Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:58.825398554Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:58.914891123Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_d8a1310d-09c1-4844-91b7-509f93b30cfd","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:58.91900179Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_27c74baf-8069-4296-ac26-2f4818d2ab3a","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:58.923309142Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_c168b27b-f467-4350-a348-4006ce85a213","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:58.927733707Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_15be01e5-1765-45f0-b521-da99b395bdb3","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:58.932269005Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_cockroachdb_ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:58.937032848Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_d8a1310d-09c1-4844-91b7-509f93b30cfd","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:58.941356621Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_27c74baf-8069-4296-ac26-2f4818d2ab3a","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:58.946111975Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_c168b27b-f467-4350-a348-4006ce85a213","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:58.950686754Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_15be01e5-1765-45f0-b521-da99b395bdb3","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:44:58.955410797Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_cockroachdb_c168b27b-f467-4350-a348-4006ce85a213:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:01.813210585Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:01.813474352Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_c168b27b-f467-4350-a348-4006ce85a213","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Profile for oxz_cockroachdb_d8a1310d-09c1-4844-91b7-509f93b30cfd:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:01.820173216Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:01.820475424Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_d8a1310d-09c1-4844-91b7-509f93b30cfd","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Profile for oxz_cockroachdb_27c74baf-8069-4296-ac26-2f4818d2ab3a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:01.830590717Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:01.83071563Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_27c74baf-8069-4296-ac26-2f4818d2ab3a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Profile for oxz_cockroachdb_15be01e5-1765-45f0-b521-da99b395bdb3:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:01.83685718Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_cockroachdb_ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:01.8372322Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:01.83724463Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_15be01e5-1765-45f0-b521-da99b395bdb3","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:01.84263045Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"wait for service Some(\"oxz_cockroachdb_c168b27b-f467-4350-a348-4006ce85a213\") failed: Property not found. retry in 63.930396ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:01.960335493Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_c168b27b-f467-4350-a348-4006ce85a213","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d8a1310d-09c1-4844-91b7-509f93b30cfd\") failed: Property not found. retry in 52.612946ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:01.968594597Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_d8a1310d-09c1-4844-91b7-509f93b30cfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_27c74baf-8069-4296-ac26-2f4818d2ab3a\") failed: Property not found. retry in 35.213877ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:01.976581425Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_27c74baf-8069-4296-ac26-2f4818d2ab3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_15be01e5-1765-45f0-b521-da99b395bdb3\") failed: Property not found. retry in 35.034651ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:01.983028693Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_15be01e5-1765-45f0-b521-da99b395bdb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0\") failed: Property not found. retry in 56.745125ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:01.989383038Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_27c74baf-8069-4296-ac26-2f4818d2ab3a\") failed: Property not found. retry in 96.743925ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:02.018892926Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_27c74baf-8069-4296-ac26-2f4818d2ab3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_15be01e5-1765-45f0-b521-da99b395bdb3\") failed: Property not found. retry in 84.50005ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:02.037080709Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_15be01e5-1765-45f0-b521-da99b395bdb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d8a1310d-09c1-4844-91b7-509f93b30cfd\") failed: Property not found. retry in 76.390393ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:02.214404373Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_d8a1310d-09c1-4844-91b7-509f93b30cfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_27c74baf-8069-4296-ac26-2f4818d2ab3a\") failed: Property not found. retry in 208.272695ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:02.29500428Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_27c74baf-8069-4296-ac26-2f4818d2ab3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c168b27b-f467-4350-a348-4006ce85a213\") failed: Property not found. retry in 109.733936ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:02.457280722Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_c168b27b-f467-4350-a348-4006ce85a213","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_15be01e5-1765-45f0-b521-da99b395bdb3\") failed: Property not found. retry in 271.428436ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:02.606135424Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_15be01e5-1765-45f0-b521-da99b395bdb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0\") failed: Property not found. retry in 142.397986ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:02.806391875Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d8a1310d-09c1-4844-91b7-509f93b30cfd\") failed: Property not found. retry in 147.50577ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:02.996046539Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_d8a1310d-09c1-4844-91b7-509f93b30cfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_27c74baf-8069-4296-ac26-2f4818d2ab3a\") failed: Property not found. retry in 489.167177ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:03.21631731Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_27c74baf-8069-4296-ac26-2f4818d2ab3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c168b27b-f467-4350-a348-4006ce85a213\") failed: Property not found. retry in 111.338854ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:03.428598773Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_c168b27b-f467-4350-a348-4006ce85a213","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_15be01e5-1765-45f0-b521-da99b395bdb3\") failed: Property not found. retry in 450.037456ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:03.787821608Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_15be01e5-1765-45f0-b521-da99b395bdb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0\") failed: Property not found. retry in 220.269867ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:04.93407912Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d8a1310d-09c1-4844-91b7-509f93b30cfd\") failed: Property not found. retry in 431.367451ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:04.954132232Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_d8a1310d-09c1-4844-91b7-509f93b30cfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_27c74baf-8069-4296-ac26-2f4818d2ab3a\") failed: Property not found. retry in 418.393264ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:04.971896214Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_27c74baf-8069-4296-ac26-2f4818d2ab3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c168b27b-f467-4350-a348-4006ce85a213\") failed: Property not found. retry in 458.592722ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:04.980876778Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_c168b27b-f467-4350-a348-4006ce85a213","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_15be01e5-1765-45f0-b521-da99b395bdb3\") failed: Property not found. retry in 787.323773ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:04.991432822Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_15be01e5-1765-45f0-b521-da99b395bdb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0\") failed: Property not found. retry in 532.530582ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:05.164166556Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d8a1310d-09c1-4844-91b7-509f93b30cfd\") failed: Property not found. retry in 465.74801ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:05.397360373Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_d8a1310d-09c1-4844-91b7-509f93b30cfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_27c74baf-8069-4296-ac26-2f4818d2ab3a\") failed: Property not found. retry in 533.012018ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:05.407552418Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_27c74baf-8069-4296-ac26-2f4818d2ab3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c168b27b-f467-4350-a348-4006ce85a213\") failed: Property not found. retry in 715.095706ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:05.451669106Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_c168b27b-f467-4350-a348-4006ce85a213","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0\") failed: Property not found. retry in 977.976353ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:05.708093027Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_15be01e5-1765-45f0-b521-da99b395bdb3\") failed: Property not found. retry in 1.275006675s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:05.789330061Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_15be01e5-1765-45f0-b521-da99b395bdb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d8a1310d-09c1-4844-91b7-509f93b30cfd\") failed: Property not found. retry in 534.993388ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:05.873913142Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_d8a1310d-09c1-4844-91b7-509f93b30cfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_27c74baf-8069-4296-ac26-2f4818d2ab3a\") failed: Property not found. retry in 780.493332ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:05.950802832Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_27c74baf-8069-4296-ac26-2f4818d2ab3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c168b27b-f467-4350-a348-4006ce85a213\") failed: Property not found. retry in 1.186094682s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:06.177638494Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_c168b27b-f467-4350-a348-4006ce85a213","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d8a1310d-09c1-4844-91b7-509f93b30cfd\") failed: Property not found. retry in 1.296585222s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:06.4185073Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_d8a1310d-09c1-4844-91b7-509f93b30cfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0\") failed: Property not found. retry in 815.863224ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:06.695846416Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_27c74baf-8069-4296-ac26-2f4818d2ab3a\") failed: Property not found. retry in 1.148301121s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:06.74174785Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_27c74baf-8069-4296-ac26-2f4818d2ab3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_15be01e5-1765-45f0-b521-da99b395bdb3\") failed: Property not found. retry in 1.493401336s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:07.074845836Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_15be01e5-1765-45f0-b521-da99b395bdb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c168b27b-f467-4350-a348-4006ce85a213\") failed: Property not found. retry in 833.469833ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:07.374561914Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_c168b27b-f467-4350-a348-4006ce85a213","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0\") failed: Property not found. retry in 658.426474ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:07.521754303Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d8a1310d-09c1-4844-91b7-509f93b30cfd\") failed: Property not found. retry in 1.429889758s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:07.726890741Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_d8a1310d-09c1-4844-91b7-509f93b30cfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_27c74baf-8069-4296-ac26-2f4818d2ab3a\") failed: Property not found. retry in 911.430388ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:07.904915062Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_27c74baf-8069-4296-ac26-2f4818d2ab3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0\") failed: Property not found. retry in 1.199340696s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:08.189965917Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c168b27b-f467-4350-a348-4006ce85a213\") failed: Property not found. retry in 1.123499225s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:08.219210218Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_c168b27b-f467-4350-a348-4006ce85a213","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_15be01e5-1765-45f0-b521-da99b395bdb3\") failed: Property not found. retry in 649.209682ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:08.577279344Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_15be01e5-1765-45f0-b521-da99b395bdb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_27c74baf-8069-4296-ac26-2f4818d2ab3a\") failed: Property not found. retry in 1.470089415s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:08.825098422Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_27c74baf-8069-4296-ac26-2f4818d2ab3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d8a1310d-09c1-4844-91b7-509f93b30cfd\") failed: Property not found. retry in 690.997896ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:09.164279486Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_d8a1310d-09c1-4844-91b7-509f93b30cfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_15be01e5-1765-45f0-b521-da99b395bdb3\") failed: Property not found. retry in 711.730983ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:09.23437662Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_15be01e5-1765-45f0-b521-da99b395bdb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c168b27b-f467-4350-a348-4006ce85a213\") failed: Property not found. retry in 812.955809ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:09.350759338Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_c168b27b-f467-4350-a348-4006ce85a213","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0\") failed: Property not found. retry in 1.494394545s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:09.397834582Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d8a1310d-09c1-4844-91b7-509f93b30cfd\") failed: Property not found. retry in 586.117533ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:09.866797854Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_d8a1310d-09c1-4844-91b7-509f93b30cfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_15be01e5-1765-45f0-b521-da99b395bdb3\") failed: Property not found. retry in 1.141115546s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:09.956779545Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_15be01e5-1765-45f0-b521-da99b395bdb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c168b27b-f467-4350-a348-4006ce85a213\") failed: Property not found. retry in 651.955617ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:10.173493612Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_c168b27b-f467-4350-a348-4006ce85a213","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_27c74baf-8069-4296-ac26-2f4818d2ab3a\") failed: Property not found. retry in 1.400502517s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:10.304729336Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_27c74baf-8069-4296-ac26-2f4818d2ab3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d8a1310d-09c1-4844-91b7-509f93b30cfd\") failed: Property not found. retry in 601.53495ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:10.462284955Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_d8a1310d-09c1-4844-91b7-509f93b30cfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c168b27b-f467-4350-a348-4006ce85a213\") failed: Property not found. retry in 761.751123ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:10.836184933Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_c168b27b-f467-4350-a348-4006ce85a213","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0\") failed: Property not found. retry in 1.357184568s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:10.902090298Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d8a1310d-09c1-4844-91b7-509f93b30cfd\") failed: Property not found. retry in 929.146512ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:11.07515112Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_d8a1310d-09c1-4844-91b7-509f93b30cfd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_15be01e5-1765-45f0-b521-da99b395bdb3\") failed: Property not found. retry in 1.320888348s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:11.106875596Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_15be01e5-1765-45f0-b521-da99b395bdb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:11.326797807Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"309.766481ms"} {"msg":"wait for service Some(\"oxz_cockroachdb_27c74baf-8069-4296-ac26-2f4818d2ab3a\") failed: Property not found. retry in 1.304291706s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:11.716371853Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_27c74baf-8069-4296-ac26-2f4818d2ab3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0\") failed: Property not found. retry in 635.253597ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:12.270654694Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0\") failed: Property not found. retry in 837.767807ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:12.916823335Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_27c74baf-8069-4296-ac26-2f4818d2ab3a\") failed: Property not found. retry in 1.088558554s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:13.03156564Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_27c74baf-8069-4296-ac26-2f4818d2ab3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0\") failed: Property not found. retry in 863.197083ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:13.939267386Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:13.972171782Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:48585"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:13.972266525Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"daed91d8-34be-42a7-918e-ee590d137ac0","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:48585","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":119,"response_code":"200"} {"msg":"wait for service Some(\"oxz_cockroachdb_27c74baf-8069-4296-ac26-2f4818d2ab3a\") failed: Property not found. retry in 1.43789946s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:14.13317602Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_27c74baf-8069-4296-ac26-2f4818d2ab3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0\") failed: Property not found. retry in 1.344516934s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:14.814467136Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_27c74baf-8069-4296-ac26-2f4818d2ab3a\") failed: Property not found. retry in 1.384384924s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:15.58297042Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_27c74baf-8069-4296-ac26-2f4818d2ab3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0\") failed: Property not found. retry in 1.175372149s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:16.168423721Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_27c74baf-8069-4296-ac26-2f4818d2ab3a\") failed: Property not found. retry in 1.048625767s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:16.977907602Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_27c74baf-8069-4296-ac26-2f4818d2ab3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0\") failed: Property not found. retry in 1.433989244s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:17.353694888Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_27c74baf-8069-4296-ac26-2f4818d2ab3a\") failed: Property not found. retry in 801.913294ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:18.036608376Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_27c74baf-8069-4296-ac26-2f4818d2ab3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0\") failed: Property not found. retry in 1.37179846s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:18.886017475Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_27c74baf-8069-4296-ac26-2f4818d2ab3a\") failed: Property not found. retry in 1.339428903s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:19.034644002Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_27c74baf-8069-4296-ac26-2f4818d2ab3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0\") failed: Property not found. retry in 558.136282ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:20.265548986Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_27c74baf-8069-4296-ac26-2f4818d2ab3a\") failed: Property not found. retry in 1.490319513s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:20.382392086Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_27c74baf-8069-4296-ac26-2f4818d2ab3a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0\") failed: Property not found. retry in 1.434698707s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:20.832129527Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:22.288961839Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_cockroachdb_15be01e5-1765-45f0-b521-da99b395bdb3"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:22.289071342Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_cockroachdb_ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:22.289087352Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_cockroachdb_d8a1310d-09c1-4844-91b7-509f93b30cfd"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:22.289101372Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_cockroachdb_27c74baf-8069-4296-ac26-2f4818d2ab3a"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:22.289113393Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_cockroachdb_c168b27b-f467-4350-a348-4006ce85a213"} {"msg":"Writing ledger to /pool/int/301424fa-c483-4ce9-92fb-3c8ee10e6f9f/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:22.289129453Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/20dccfdd-e68a-468d-8629-7f7e0834aa1c/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:22.289578945Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:22.290224172Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"d218f235-00fb-4a12-949a-28430e9b3d5b","remote_addr":"[fd00:1122:3344:101::1]:54642","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":24041550,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:22.324613436Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:38809"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::5]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:22.351384343Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3403"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:22.709633143Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3421"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:26.67386756Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"994.43046ms"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:42.703285794Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"1.203622472s"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:44.008119301Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:55204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:44.008186103Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"814b755f-b4ab-4b80-807b-7a505171eeb5","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:55204","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":40,"response_code":"200"} {"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:44.740862075Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3444"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:44.740910416Z","hostname":"buskin","pid":1555,"uri":"/cockroachdb","method":"POST","req_id":"364d84ca-532d-4e5d-aca9-4a5120bef68c","remote_addr":"[fd00:1122:3344:101::1]:38809","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":22416303,"response_code":"204"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 8ae14f97-bd01-42ad-87d5-23eae9194521, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: bc60810c-eece-4cbd-a0f8-d04271924b7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bf3fc707-b452-4d30-a92b-e56b4d4122ee, kind: Service { id: bc60810c-eece-4cbd-a0f8-d04271924b7c }, name: Name(\"ntp-bc60810c-eece-4cbd-a0f8-d04271924b7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 196])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: c168b27b-f467-4350-a348-4006ce85a213, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 15be01e5-1765-45f0-b521-da99b395bdb3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: d8a1310d-09c1-4844-91b7-509f93b30cfd, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 59731bd7-ffc9-493f-a33c-a1cf5ba2df30, kind: Service { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1 }, name: Name(\"external-dns-1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 32])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 09fdc6fd-8297-436e-903d-b18710599203, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 005b2740-84d5-473e-ae59-aa79fd2fe5b6, kind: Service { id: 09fdc6fd-8297-436e-903d-b18710599203 }, name: Name(\"external-dns-09fdc6fd-8297-436e-903d-b18710599203\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: c6edf68c-f558-4f51-9aed-6eb91fb3ad51, kind: Service { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52 }, name: Name(\"nexus-87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 117])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6088410e-a8fc-4d33-b593-6385e863291f, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e6898038-bf11-41c8-b971-6ff3eed6d38b, kind: Service { id: 6088410e-a8fc-4d33-b593-6385e863291f }, name: Name(\"nexus-6088410e-a8fc-4d33-b593-6385e863291f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 27])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 742e0330-77de-4723-a145-74751d41c73b, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: ffe6d41d-1349-42b2-a887-49d734eef422, kind: Service { id: 742e0330-77de-4723-a145-74751d41c73b }, name: Name(\"nexus-742e0330-77de-4723-a145-74751d41c73b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8d644923-5967-4351-a1ad-fe8a938f33e1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 740bd3e0-b30c-4033-968a-5451c8d5adcd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3286a4ca-fffd-4ed0-9c21-d528e1d23603, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b1925c8-3ae5-4372-a3de-213551574fe1, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 663bd1f7-7a29-484c-bfce-d312646b1660, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: c6d325c3-cddf-4528-8ba6-a083f6511284, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External } } } }, OmicronZoneConfig { id: c30b7bc1-fd10-4942-8055-ada48d2e3957, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External } } } }, OmicronZoneConfig { id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:44.772278072Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:44.772440207Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:37752"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:44.772827106Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:44.772838927Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:44.79833586Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:44.798391352Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:44.820343623Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:44.820391924Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:44.839257845Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:44.839308166Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:44.859342168Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:44.859395879Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:44.879341988Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:44.879392949Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:44.899936964Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:44.899987135Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:44.919519623Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:44.919576604Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:44.93900437Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:44.939044891Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:45.028730444Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:45.028785026Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:45.109738432Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:45.109796273Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:45.197015913Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:45.197073034Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:45.286902111Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:45.286965963Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:45.358299449Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:45.35835162Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:45.446164175Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:45.446223816Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:45.527877811Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:45.527928312Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:45.599457983Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:45.599512965Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:45.672533044Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:45.672583786Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:45.750199125Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:45.750244586Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:45.831388097Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:45.831445489Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:45.908580735Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/301424fa-c483-4ce9-92fb-3c8ee10e6f9f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:45.908639767Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/20dccfdd-e68a-468d-8629-7f7e0834aa1c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:45.908995796Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:45.909191631Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:46.015029945Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:46.015079346Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:46.015095497Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:46.015100386Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:46.015107447Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:46.015112947Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:46.015118357Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:46.015125857Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:46.015132787Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:46.015138307Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:46.015143708Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:46.015150738Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:46.015156438Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:46.015161578Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:46.015166888Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:46.015173519Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:46.015179319Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:46.015200309Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:46.015205899Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.015312682Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"947ca388-168b-44e3-ae08-ffc91da4dace","remote_addr":"[fd00:1122:3344:101::1]:37752","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b\", TimeNotSynchronized), (\"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17\", TimeNotSynchronized), (\"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\", TimeNotSynchronized), (\"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203\", TimeNotSynchronized), (\"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1\", TimeNotSynchronized), (\"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1\", TimeNotSynchronized), (\"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1\", TimeNotSynchronized), (\"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9\", TimeNotSynchronized), (\"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957\", TimeNotSynchronized), (\"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7\", TimeNotSynchronized), (\"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603\", TimeNotSynchronized), (\"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f\", TimeNotSynchronized), (\"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec\", TimeNotSynchronized), (\"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\", TimeNotSynchronized), (\"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd\", TimeNotSynchronized), (\"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660\", TimeNotSynchronized), (\"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb\", TimeNotSynchronized), (\"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b\", TimeNotSynchronized), (\"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284\", TimeNotSynchronized)]","latency_us":1242742,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:46.01562526Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"101.356559ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"947ca388-168b-44e3-ae08-ffc91da4dace\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 17:45:46 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"947ca388-168b-44e3-ae08-ffc91da4dace\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 8ae14f97-bd01-42ad-87d5-23eae9194521, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: bc60810c-eece-4cbd-a0f8-d04271924b7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bf3fc707-b452-4d30-a92b-e56b4d4122ee, kind: Service { id: bc60810c-eece-4cbd-a0f8-d04271924b7c }, name: Name(\"ntp-bc60810c-eece-4cbd-a0f8-d04271924b7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 196])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: c168b27b-f467-4350-a348-4006ce85a213, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 15be01e5-1765-45f0-b521-da99b395bdb3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: d8a1310d-09c1-4844-91b7-509f93b30cfd, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 59731bd7-ffc9-493f-a33c-a1cf5ba2df30, kind: Service { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1 }, name: Name(\"external-dns-1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 32])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 09fdc6fd-8297-436e-903d-b18710599203, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 005b2740-84d5-473e-ae59-aa79fd2fe5b6, kind: Service { id: 09fdc6fd-8297-436e-903d-b18710599203 }, name: Name(\"external-dns-09fdc6fd-8297-436e-903d-b18710599203\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: c6edf68c-f558-4f51-9aed-6eb91fb3ad51, kind: Service { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52 }, name: Name(\"nexus-87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 117])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6088410e-a8fc-4d33-b593-6385e863291f, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e6898038-bf11-41c8-b971-6ff3eed6d38b, kind: Service { id: 6088410e-a8fc-4d33-b593-6385e863291f }, name: Name(\"nexus-6088410e-a8fc-4d33-b593-6385e863291f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 27])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 742e0330-77de-4723-a145-74751d41c73b, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: ffe6d41d-1349-42b2-a887-49d734eef422, kind: Service { id: 742e0330-77de-4723-a145-74751d41c73b }, name: Name(\"nexus-742e0330-77de-4723-a145-74751d41c73b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8d644923-5967-4351-a1ad-fe8a938f33e1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 740bd3e0-b30c-4033-968a-5451c8d5adcd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3286a4ca-fffd-4ed0-9c21-d528e1d23603, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b1925c8-3ae5-4372-a3de-213551574fe1, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 663bd1f7-7a29-484c-bfce-d312646b1660, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: c6d325c3-cddf-4528-8ba6-a083f6511284, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External } } } }, OmicronZoneConfig { id: c30b7bc1-fd10-4942-8055-ada48d2e3957, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External } } } }, OmicronZoneConfig { id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.117905221Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.118525977Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.118538388Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.141370432Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.141417133Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.162323837Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.162372568Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.179466513Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.179520254Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.197214855Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.197263926Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.214634268Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.214677179Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.233710154Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.233758415Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.251814785Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.251859656Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.270704477Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.270753498Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.292704849Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.29274867Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.312597326Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.312644128Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.333920801Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.333968573Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.353450249Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.353500951Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.371836378Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.371889949Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.392010683Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.392053114Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.409782535Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.409831746Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.426364977Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.426414058Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.442446615Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.442492896Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.460930816Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.460985897Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.477204729Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.47724988Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.494887909Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/301424fa-c483-4ce9-92fb-3c8ee10e6f9f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.494950281Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/20dccfdd-e68a-468d-8629-7f7e0834aa1c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.495198677Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.495367742Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:46.589541832Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:46.589592883Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:46.589599583Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:46.589605284Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:46.589609474Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:46.589624644Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:46.589632024Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:46.589637964Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:46.589645105Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:46.589651385Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:46.589656895Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:46.589663805Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:46.589669885Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:46.589675525Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:46.589679655Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:46.589685316Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:46.589690656Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:46.589696326Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:46.589701946Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.58986145Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"6381de54-29df-4b3c-87e0-ce76aecdb3cb","remote_addr":"[fd00:1122:3344:101::1]:37752","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1\", TimeNotSynchronized), (\"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957\", TimeNotSynchronized), (\"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17\", TimeNotSynchronized), (\"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb\", TimeNotSynchronized), (\"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660\", TimeNotSynchronized), (\"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203\", TimeNotSynchronized), (\"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd\", TimeNotSynchronized), (\"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9\", TimeNotSynchronized), (\"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\", TimeNotSynchronized), (\"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284\", TimeNotSynchronized), (\"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b\", TimeNotSynchronized), (\"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603\", TimeNotSynchronized), (\"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7\", TimeNotSynchronized), (\"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\", TimeNotSynchronized), (\"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1\", TimeNotSynchronized), (\"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f\", TimeNotSynchronized), (\"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b\", TimeNotSynchronized), (\"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec\", TimeNotSynchronized), (\"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1\", TimeNotSynchronized)]","latency_us":471622,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:46.590079366Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"152.690002ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6381de54-29df-4b3c-87e0-ce76aecdb3cb\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 17:45:46 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6381de54-29df-4b3c-87e0-ce76aecdb3cb\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 8ae14f97-bd01-42ad-87d5-23eae9194521, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: bc60810c-eece-4cbd-a0f8-d04271924b7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bf3fc707-b452-4d30-a92b-e56b4d4122ee, kind: Service { id: bc60810c-eece-4cbd-a0f8-d04271924b7c }, name: Name(\"ntp-bc60810c-eece-4cbd-a0f8-d04271924b7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 196])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: c168b27b-f467-4350-a348-4006ce85a213, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 15be01e5-1765-45f0-b521-da99b395bdb3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: d8a1310d-09c1-4844-91b7-509f93b30cfd, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 59731bd7-ffc9-493f-a33c-a1cf5ba2df30, kind: Service { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1 }, name: Name(\"external-dns-1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 32])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 09fdc6fd-8297-436e-903d-b18710599203, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 005b2740-84d5-473e-ae59-aa79fd2fe5b6, kind: Service { id: 09fdc6fd-8297-436e-903d-b18710599203 }, name: Name(\"external-dns-09fdc6fd-8297-436e-903d-b18710599203\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: c6edf68c-f558-4f51-9aed-6eb91fb3ad51, kind: Service { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52 }, name: Name(\"nexus-87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 117])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6088410e-a8fc-4d33-b593-6385e863291f, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e6898038-bf11-41c8-b971-6ff3eed6d38b, kind: Service { id: 6088410e-a8fc-4d33-b593-6385e863291f }, name: Name(\"nexus-6088410e-a8fc-4d33-b593-6385e863291f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 27])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 742e0330-77de-4723-a145-74751d41c73b, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: ffe6d41d-1349-42b2-a887-49d734eef422, kind: Service { id: 742e0330-77de-4723-a145-74751d41c73b }, name: Name(\"nexus-742e0330-77de-4723-a145-74751d41c73b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8d644923-5967-4351-a1ad-fe8a938f33e1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 740bd3e0-b30c-4033-968a-5451c8d5adcd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3286a4ca-fffd-4ed0-9c21-d528e1d23603, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b1925c8-3ae5-4372-a3de-213551574fe1, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 663bd1f7-7a29-484c-bfce-d312646b1660, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: c6d325c3-cddf-4528-8ba6-a083f6511284, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External } } } }, OmicronZoneConfig { id: c30b7bc1-fd10-4942-8055-ada48d2e3957, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External } } } }, OmicronZoneConfig { id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.743308512Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.743726503Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.743745194Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.768465157Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.768517698Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.789835023Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.789890475Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.811955069Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.81200415Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.832438051Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.832487603Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.852299508Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.85235728Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.872169255Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.872219316Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.893530861Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.893584782Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.91427672Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.914323312Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.939948939Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.94001292Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.962929286Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.962980688Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.983363338Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:46.983417239Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.002335701Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.002381922Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.024565289Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.024619241Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.046825318Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.04689009Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.06843403Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.068488312Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.08994702Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.090000211Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.110716641Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.110775482Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.132863517Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.132909648Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.153707589Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.153775491Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.174781367Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/301424fa-c483-4ce9-92fb-3c8ee10e6f9f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.174830929Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/20dccfdd-e68a-468d-8629-7f7e0834aa1c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.175042104Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.175193828Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:47.273896816Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:47.273969078Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:47.273977668Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:47.273983648Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:47.273989588Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:47.273996769Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:47.274002869Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:47.274010479Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:47.274017009Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:47.274031859Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:47.27403952Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:47.2740457Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:47.274052Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:47.27405918Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:47.27406533Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:47.27407092Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:47.274076511Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:47.274083391Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:47.274103161Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.274264245Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"195d9923-8011-4b0a-8dcb-60959b54ea3a","remote_addr":"[fd00:1122:3344:101::1]:37752","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b\", TimeNotSynchronized), (\"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec\", TimeNotSynchronized), (\"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7\", TimeNotSynchronized), (\"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9\", TimeNotSynchronized), (\"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\", TimeNotSynchronized), (\"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203\", TimeNotSynchronized), (\"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1\", TimeNotSynchronized), (\"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b\", TimeNotSynchronized), (\"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1\", TimeNotSynchronized), (\"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660\", TimeNotSynchronized), (\"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f\", TimeNotSynchronized), (\"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\", TimeNotSynchronized), (\"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603\", TimeNotSynchronized), (\"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1\", TimeNotSynchronized), (\"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284\", TimeNotSynchronized), (\"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb\", TimeNotSynchronized), (\"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd\", TimeNotSynchronized), (\"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17\", TimeNotSynchronized), (\"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957\", TimeNotSynchronized)]","latency_us":530769,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:47.274529622Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"188.139096ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"195d9923-8011-4b0a-8dcb-60959b54ea3a\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 17:45:47 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"195d9923-8011-4b0a-8dcb-60959b54ea3a\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 8ae14f97-bd01-42ad-87d5-23eae9194521, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: bc60810c-eece-4cbd-a0f8-d04271924b7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bf3fc707-b452-4d30-a92b-e56b4d4122ee, kind: Service { id: bc60810c-eece-4cbd-a0f8-d04271924b7c }, name: Name(\"ntp-bc60810c-eece-4cbd-a0f8-d04271924b7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 196])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: c168b27b-f467-4350-a348-4006ce85a213, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 15be01e5-1765-45f0-b521-da99b395bdb3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: d8a1310d-09c1-4844-91b7-509f93b30cfd, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 59731bd7-ffc9-493f-a33c-a1cf5ba2df30, kind: Service { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1 }, name: Name(\"external-dns-1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 32])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 09fdc6fd-8297-436e-903d-b18710599203, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 005b2740-84d5-473e-ae59-aa79fd2fe5b6, kind: Service { id: 09fdc6fd-8297-436e-903d-b18710599203 }, name: Name(\"external-dns-09fdc6fd-8297-436e-903d-b18710599203\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: c6edf68c-f558-4f51-9aed-6eb91fb3ad51, kind: Service { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52 }, name: Name(\"nexus-87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 117])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6088410e-a8fc-4d33-b593-6385e863291f, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e6898038-bf11-41c8-b971-6ff3eed6d38b, kind: Service { id: 6088410e-a8fc-4d33-b593-6385e863291f }, name: Name(\"nexus-6088410e-a8fc-4d33-b593-6385e863291f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 27])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 742e0330-77de-4723-a145-74751d41c73b, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: ffe6d41d-1349-42b2-a887-49d734eef422, kind: Service { id: 742e0330-77de-4723-a145-74751d41c73b }, name: Name(\"nexus-742e0330-77de-4723-a145-74751d41c73b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8d644923-5967-4351-a1ad-fe8a938f33e1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 740bd3e0-b30c-4033-968a-5451c8d5adcd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3286a4ca-fffd-4ed0-9c21-d528e1d23603, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b1925c8-3ae5-4372-a3de-213551574fe1, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 663bd1f7-7a29-484c-bfce-d312646b1660, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: c6d325c3-cddf-4528-8ba6-a083f6511284, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External } } } }, OmicronZoneConfig { id: c30b7bc1-fd10-4942-8055-ada48d2e3957, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External } } } }, OmicronZoneConfig { id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.463781366Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.464129015Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.464137136Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.488082468Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.48813332Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.509565517Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.509613699Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.529136956Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.529185928Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.549799594Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.549848495Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.569524807Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.569571859Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.58960486Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.589659311Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.608349797Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.608397019Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.628999105Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.629062846Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.648926723Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.648980205Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.6684006Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.668451061Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.688815401Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.688859982Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.708078262Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.708121663Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.728263557Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.728311718Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.747855827Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.747899178Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.767616641Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.767666142Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.78680246Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.786848312Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.807116159Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.80716639Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.828210598Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.828263889Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.848915226Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.848963197Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.869448321Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/301424fa-c483-4ce9-92fb-3c8ee10e6f9f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.869505112Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/20dccfdd-e68a-468d-8629-7f7e0834aa1c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.869725908Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.869874732Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:47.964706119Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:47.96476462Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:47.964773431Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:47.964779191Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:47.964786301Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:47.964792171Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:47.964797721Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:47.964802991Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:47.964808422Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:47.964813732Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:47.964819162Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:47.964825752Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:47.964831432Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:47.964847463Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:47.964854513Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:47.964858413Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:47.964861963Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:47.964867383Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:47.964870963Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:47.964959575Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"bdaca22f-e672-4895-8223-0883f4c48399","remote_addr":"[fd00:1122:3344:101::1]:37752","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1\", TimeNotSynchronized), (\"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec\", TimeNotSynchronized), (\"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b\", TimeNotSynchronized), (\"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\", TimeNotSynchronized), (\"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd\", TimeNotSynchronized), (\"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1\", TimeNotSynchronized), (\"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17\", TimeNotSynchronized), (\"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203\", TimeNotSynchronized), (\"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284\", TimeNotSynchronized), (\"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b\", TimeNotSynchronized), (\"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb\", TimeNotSynchronized), (\"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\", TimeNotSynchronized), (\"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660\", TimeNotSynchronized), (\"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603\", TimeNotSynchronized), (\"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1\", TimeNotSynchronized), (\"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9\", TimeNotSynchronized), (\"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7\", TimeNotSynchronized), (\"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957\", TimeNotSynchronized), (\"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f\", TimeNotSynchronized)]","latency_us":501034,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:47.965206572Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"163.372244ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"bdaca22f-e672-4895-8223-0883f4c48399\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 17:45:47 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"bdaca22f-e672-4895-8223-0883f4c48399\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 8ae14f97-bd01-42ad-87d5-23eae9194521, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: bc60810c-eece-4cbd-a0f8-d04271924b7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bf3fc707-b452-4d30-a92b-e56b4d4122ee, kind: Service { id: bc60810c-eece-4cbd-a0f8-d04271924b7c }, name: Name(\"ntp-bc60810c-eece-4cbd-a0f8-d04271924b7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 196])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: c168b27b-f467-4350-a348-4006ce85a213, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 15be01e5-1765-45f0-b521-da99b395bdb3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: d8a1310d-09c1-4844-91b7-509f93b30cfd, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 59731bd7-ffc9-493f-a33c-a1cf5ba2df30, kind: Service { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1 }, name: Name(\"external-dns-1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 32])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 09fdc6fd-8297-436e-903d-b18710599203, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 005b2740-84d5-473e-ae59-aa79fd2fe5b6, kind: Service { id: 09fdc6fd-8297-436e-903d-b18710599203 }, name: Name(\"external-dns-09fdc6fd-8297-436e-903d-b18710599203\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: c6edf68c-f558-4f51-9aed-6eb91fb3ad51, kind: Service { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52 }, name: Name(\"nexus-87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 117])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6088410e-a8fc-4d33-b593-6385e863291f, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e6898038-bf11-41c8-b971-6ff3eed6d38b, kind: Service { id: 6088410e-a8fc-4d33-b593-6385e863291f }, name: Name(\"nexus-6088410e-a8fc-4d33-b593-6385e863291f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 27])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 742e0330-77de-4723-a145-74751d41c73b, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: ffe6d41d-1349-42b2-a887-49d734eef422, kind: Service { id: 742e0330-77de-4723-a145-74751d41c73b }, name: Name(\"nexus-742e0330-77de-4723-a145-74751d41c73b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8d644923-5967-4351-a1ad-fe8a938f33e1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 740bd3e0-b30c-4033-968a-5451c8d5adcd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3286a4ca-fffd-4ed0-9c21-d528e1d23603, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b1925c8-3ae5-4372-a3de-213551574fe1, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 663bd1f7-7a29-484c-bfce-d312646b1660, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: c6d325c3-cddf-4528-8ba6-a083f6511284, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External } } } }, OmicronZoneConfig { id: c30b7bc1-fd10-4942-8055-ada48d2e3957, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External } } } }, OmicronZoneConfig { id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.129728052Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.130109802Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.130117082Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.153034608Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.15308815Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.173508031Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.173554402Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.192627769Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.19267836Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.212773742Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.212823094Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.231596822Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.231642633Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.250937215Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.250982867Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.27032721Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.270377891Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.29071274Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.290757062Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.311760878Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.311802989Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.333857043Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.333908674Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.356239075Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.356277186Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.377098218Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.377150369Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.396699418Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.396739699Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.415483787Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.415526808Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.435729283Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.435774475Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.455764124Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.455805556Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.474772599Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.47481323Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.494533023Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.494576364Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.514127823Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.514173324Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.535237962Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/301424fa-c483-4ce9-92fb-3c8ee10e6f9f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.535306484Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/20dccfdd-e68a-468d-8629-7f7e0834aa1c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.535497389Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.535621472Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:48.633362305Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:48.633426597Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:48.633452647Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:48.633458568Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:48.633464248Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:48.633469698Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:48.633476538Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:48.633481808Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:48.633487308Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:48.633493188Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:48.633500439Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:48.633506199Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:48.633511779Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:48.633517639Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:48.633523139Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:48.633530269Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:48.63353466Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:48.63354556Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:48.63354966Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.633694434Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"cda63b96-b096-41d3-a3ea-0f752f756964","remote_addr":"[fd00:1122:3344:101::1]:37752","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660\", TimeNotSynchronized), (\"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1\", TimeNotSynchronized), (\"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9\", TimeNotSynchronized), (\"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203\", TimeNotSynchronized), (\"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1\", TimeNotSynchronized), (\"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb\", TimeNotSynchronized), (\"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957\", TimeNotSynchronized), (\"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\", TimeNotSynchronized), (\"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec\", TimeNotSynchronized), (\"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284\", TimeNotSynchronized), (\"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17\", TimeNotSynchronized), (\"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7\", TimeNotSynchronized), (\"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603\", TimeNotSynchronized), (\"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f\", TimeNotSynchronized), (\"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\", TimeNotSynchronized), (\"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b\", TimeNotSynchronized), (\"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b\", TimeNotSynchronized), (\"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1\", TimeNotSynchronized), (\"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd\", TimeNotSynchronized)]","latency_us":503815,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:48.63394223Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"233.201311ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"cda63b96-b096-41d3-a3ea-0f752f756964\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 17:45:48 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"cda63b96-b096-41d3-a3ea-0f752f756964\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 8ae14f97-bd01-42ad-87d5-23eae9194521, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: bc60810c-eece-4cbd-a0f8-d04271924b7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bf3fc707-b452-4d30-a92b-e56b4d4122ee, kind: Service { id: bc60810c-eece-4cbd-a0f8-d04271924b7c }, name: Name(\"ntp-bc60810c-eece-4cbd-a0f8-d04271924b7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 196])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: c168b27b-f467-4350-a348-4006ce85a213, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 15be01e5-1765-45f0-b521-da99b395bdb3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: d8a1310d-09c1-4844-91b7-509f93b30cfd, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 59731bd7-ffc9-493f-a33c-a1cf5ba2df30, kind: Service { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1 }, name: Name(\"external-dns-1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 32])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 09fdc6fd-8297-436e-903d-b18710599203, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 005b2740-84d5-473e-ae59-aa79fd2fe5b6, kind: Service { id: 09fdc6fd-8297-436e-903d-b18710599203 }, name: Name(\"external-dns-09fdc6fd-8297-436e-903d-b18710599203\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: c6edf68c-f558-4f51-9aed-6eb91fb3ad51, kind: Service { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52 }, name: Name(\"nexus-87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 117])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6088410e-a8fc-4d33-b593-6385e863291f, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e6898038-bf11-41c8-b971-6ff3eed6d38b, kind: Service { id: 6088410e-a8fc-4d33-b593-6385e863291f }, name: Name(\"nexus-6088410e-a8fc-4d33-b593-6385e863291f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 27])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 742e0330-77de-4723-a145-74751d41c73b, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: ffe6d41d-1349-42b2-a887-49d734eef422, kind: Service { id: 742e0330-77de-4723-a145-74751d41c73b }, name: Name(\"nexus-742e0330-77de-4723-a145-74751d41c73b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8d644923-5967-4351-a1ad-fe8a938f33e1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 740bd3e0-b30c-4033-968a-5451c8d5adcd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3286a4ca-fffd-4ed0-9c21-d528e1d23603, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b1925c8-3ae5-4372-a3de-213551574fe1, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 663bd1f7-7a29-484c-bfce-d312646b1660, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: c6d325c3-cddf-4528-8ba6-a083f6511284, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External } } } }, OmicronZoneConfig { id: c30b7bc1-fd10-4942-8055-ada48d2e3957, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External } } } }, OmicronZoneConfig { id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.869214861Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.869573101Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.869630192Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.893588715Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.893631727Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.915628169Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.91567613Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.935220169Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.93526908Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.954637574Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.954695865Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.973793432Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.973836983Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.991234406Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:48.991267127Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.009700715Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.009749467Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.02792236Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.027958341Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.046902223Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.046940844Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.067248103Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.067296324Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.085781955Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.085823416Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.107762357Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.107799818Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.128474305Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.128536017Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.147127011Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.147160992Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.166610758Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.166649219Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.18591608Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.185958861Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.2058999Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.205939671Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.225059268Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.225098399Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.244957486Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.244998147Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.265648194Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/301424fa-c483-4ce9-92fb-3c8ee10e6f9f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.265699766Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/20dccfdd-e68a-468d-8629-7f7e0834aa1c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.265910011Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.266049855Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:49.361690403Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:49.361744185Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:49.361751245Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:49.361758565Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:49.361764175Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:49.361769865Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:49.361794866Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:49.361800476Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:49.361806886Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:49.361812296Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:49.361818907Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:49.361822777Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:49.361826497Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:49.361831717Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:49.361835377Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:49.361839497Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:49.361844797Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:49.361848387Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:49.361853747Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.36195042Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"11cc6bb6-b6c3-489b-88b6-b7bf3caae8b9","remote_addr":"[fd00:1122:3344:101::1]:37752","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603\", TimeNotSynchronized), (\"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\", TimeNotSynchronized), (\"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\", TimeNotSynchronized), (\"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b\", TimeNotSynchronized), (\"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1\", TimeNotSynchronized), (\"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec\", TimeNotSynchronized), (\"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9\", TimeNotSynchronized), (\"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284\", TimeNotSynchronized), (\"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1\", TimeNotSynchronized), (\"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b\", TimeNotSynchronized), (\"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f\", TimeNotSynchronized), (\"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd\", TimeNotSynchronized), (\"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7\", TimeNotSynchronized), (\"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203\", TimeNotSynchronized), (\"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1\", TimeNotSynchronized), (\"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb\", TimeNotSynchronized), (\"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957\", TimeNotSynchronized), (\"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17\", TimeNotSynchronized), (\"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660\", TimeNotSynchronized)]","latency_us":492560,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:49.362173746Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"307.64155ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"11cc6bb6-b6c3-489b-88b6-b7bf3caae8b9\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 17:45:48 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"11cc6bb6-b6c3-489b-88b6-b7bf3caae8b9\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 8ae14f97-bd01-42ad-87d5-23eae9194521, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: bc60810c-eece-4cbd-a0f8-d04271924b7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bf3fc707-b452-4d30-a92b-e56b4d4122ee, kind: Service { id: bc60810c-eece-4cbd-a0f8-d04271924b7c }, name: Name(\"ntp-bc60810c-eece-4cbd-a0f8-d04271924b7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 196])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: c168b27b-f467-4350-a348-4006ce85a213, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 15be01e5-1765-45f0-b521-da99b395bdb3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: d8a1310d-09c1-4844-91b7-509f93b30cfd, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 59731bd7-ffc9-493f-a33c-a1cf5ba2df30, kind: Service { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1 }, name: Name(\"external-dns-1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 32])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 09fdc6fd-8297-436e-903d-b18710599203, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 005b2740-84d5-473e-ae59-aa79fd2fe5b6, kind: Service { id: 09fdc6fd-8297-436e-903d-b18710599203 }, name: Name(\"external-dns-09fdc6fd-8297-436e-903d-b18710599203\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: c6edf68c-f558-4f51-9aed-6eb91fb3ad51, kind: Service { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52 }, name: Name(\"nexus-87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 117])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6088410e-a8fc-4d33-b593-6385e863291f, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e6898038-bf11-41c8-b971-6ff3eed6d38b, kind: Service { id: 6088410e-a8fc-4d33-b593-6385e863291f }, name: Name(\"nexus-6088410e-a8fc-4d33-b593-6385e863291f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 27])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 742e0330-77de-4723-a145-74751d41c73b, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: ffe6d41d-1349-42b2-a887-49d734eef422, kind: Service { id: 742e0330-77de-4723-a145-74751d41c73b }, name: Name(\"nexus-742e0330-77de-4723-a145-74751d41c73b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8d644923-5967-4351-a1ad-fe8a938f33e1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 740bd3e0-b30c-4033-968a-5451c8d5adcd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3286a4ca-fffd-4ed0-9c21-d528e1d23603, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b1925c8-3ae5-4372-a3de-213551574fe1, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 663bd1f7-7a29-484c-bfce-d312646b1660, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: c6d325c3-cddf-4528-8ba6-a083f6511284, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External } } } }, OmicronZoneConfig { id: c30b7bc1-fd10-4942-8055-ada48d2e3957, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External } } } }, OmicronZoneConfig { id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.670492147Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.670844796Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.670861397Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.694224895Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.694270076Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.717955242Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.717998003Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.74017544Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.740225701Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.761325621Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.761367042Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.784193776Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.784249527Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.806487116Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.806527666Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.826880366Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.826922407Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.848380615Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.848421496Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.869436593Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.869479164Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.891526248Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.891572339Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.913748926Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.913794447Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.935242195Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.935285456Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.956366475Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.956408846Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.977433573Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.977477194Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.997375352Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:49.997418653Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.018336417Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.018382298Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.041306175Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.041351986Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.062289341Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.062329672Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.084074767Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.084120939Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.10533272Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/301424fa-c483-4ce9-92fb-3c8ee10e6f9f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.105388192Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/20dccfdd-e68a-468d-8629-7f7e0834aa1c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.105581127Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.10571833Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:50.206165924Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:50.206235256Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:50.206242516Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:50.206248556Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:50.206252876Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:50.206258936Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:50.206264837Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:50.206270277Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:50.206275967Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:50.206301667Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:50.206309248Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:50.206314848Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:50.206320598Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:50.206326208Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:50.206331618Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:50.206336848Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:50.206343719Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:50.206347519Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:50.206352749Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.206442261Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"5384f9aa-45a9-4a9c-8f3e-4cb8308fef4d","remote_addr":"[fd00:1122:3344:101::1]:37752","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1\", TimeNotSynchronized), (\"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\", TimeNotSynchronized), (\"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd\", TimeNotSynchronized), (\"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203\", TimeNotSynchronized), (\"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603\", TimeNotSynchronized), (\"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9\", TimeNotSynchronized), (\"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec\", TimeNotSynchronized), (\"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660\", TimeNotSynchronized), (\"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb\", TimeNotSynchronized), (\"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b\", TimeNotSynchronized), (\"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957\", TimeNotSynchronized), (\"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17\", TimeNotSynchronized), (\"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b\", TimeNotSynchronized), (\"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1\", TimeNotSynchronized), (\"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1\", TimeNotSynchronized), (\"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284\", TimeNotSynchronized), (\"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\", TimeNotSynchronized), (\"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7\", TimeNotSynchronized), (\"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f\", TimeNotSynchronized)]","latency_us":535807,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:50.206688888Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"311.250186ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5384f9aa-45a9-4a9c-8f3e-4cb8308fef4d\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 17:45:49 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5384f9aa-45a9-4a9c-8f3e-4cb8308fef4d\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 8ae14f97-bd01-42ad-87d5-23eae9194521, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: bc60810c-eece-4cbd-a0f8-d04271924b7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bf3fc707-b452-4d30-a92b-e56b4d4122ee, kind: Service { id: bc60810c-eece-4cbd-a0f8-d04271924b7c }, name: Name(\"ntp-bc60810c-eece-4cbd-a0f8-d04271924b7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 196])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: c168b27b-f467-4350-a348-4006ce85a213, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 15be01e5-1765-45f0-b521-da99b395bdb3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: d8a1310d-09c1-4844-91b7-509f93b30cfd, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 59731bd7-ffc9-493f-a33c-a1cf5ba2df30, kind: Service { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1 }, name: Name(\"external-dns-1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 32])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 09fdc6fd-8297-436e-903d-b18710599203, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 005b2740-84d5-473e-ae59-aa79fd2fe5b6, kind: Service { id: 09fdc6fd-8297-436e-903d-b18710599203 }, name: Name(\"external-dns-09fdc6fd-8297-436e-903d-b18710599203\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: c6edf68c-f558-4f51-9aed-6eb91fb3ad51, kind: Service { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52 }, name: Name(\"nexus-87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 117])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6088410e-a8fc-4d33-b593-6385e863291f, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e6898038-bf11-41c8-b971-6ff3eed6d38b, kind: Service { id: 6088410e-a8fc-4d33-b593-6385e863291f }, name: Name(\"nexus-6088410e-a8fc-4d33-b593-6385e863291f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 27])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 742e0330-77de-4723-a145-74751d41c73b, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: ffe6d41d-1349-42b2-a887-49d734eef422, kind: Service { id: 742e0330-77de-4723-a145-74751d41c73b }, name: Name(\"nexus-742e0330-77de-4723-a145-74751d41c73b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8d644923-5967-4351-a1ad-fe8a938f33e1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 740bd3e0-b30c-4033-968a-5451c8d5adcd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3286a4ca-fffd-4ed0-9c21-d528e1d23603, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b1925c8-3ae5-4372-a3de-213551574fe1, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 663bd1f7-7a29-484c-bfce-d312646b1660, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: c6d325c3-cddf-4528-8ba6-a083f6511284, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External } } } }, OmicronZoneConfig { id: c30b7bc1-fd10-4942-8055-ada48d2e3957, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External } } } }, OmicronZoneConfig { id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.519003213Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.520060361Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.520100192Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.544760023Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.544805244Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.566800707Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.566844708Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.584300322Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.584337093Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.601581151Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.601623023Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.618972614Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.619009885Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.637526017Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.637582528Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.659154469Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.659200021Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.680130715Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.680168006Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.700886995Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.700926496Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.722872707Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.722916458Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.745688451Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.745738122Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.769155801Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.769204763Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.789233294Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.789276525Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.808868464Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.808908396Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.832164481Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.832210422Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.853019343Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.853063004Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.873206278Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.873243519Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.894152223Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.894193595Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.915720855Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.915764136Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.935640273Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/301424fa-c483-4ce9-92fb-3c8ee10e6f9f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.935701134Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/20dccfdd-e68a-468d-8629-7f7e0834aa1c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.93591562Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:50.936058474Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:51.034220847Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:51.034281059Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:51.034288589Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:51.034296049Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:51.034302149Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:51.03430814Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:51.03431545Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:51.03432109Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:51.03432696Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:51.03433398Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:51.03433989Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:51.034346081Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:51.034353401Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:51.034377031Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:51.034384442Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:51.034390322Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:51.034395742Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:51.034402672Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:51.034406782Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:51.034548136Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"5a03f9ca-d7b8-4beb-bc7b-6fc87f940456","remote_addr":"[fd00:1122:3344:101::1]:37752","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1\", TimeNotSynchronized), (\"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b\", TimeNotSynchronized), (\"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd\", TimeNotSynchronized), (\"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1\", TimeNotSynchronized), (\"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9\", TimeNotSynchronized), (\"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7\", TimeNotSynchronized), (\"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17\", TimeNotSynchronized), (\"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603\", TimeNotSynchronized), (\"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f\", TimeNotSynchronized), (\"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b\", TimeNotSynchronized), (\"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\", TimeNotSynchronized), (\"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660\", TimeNotSynchronized), (\"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb\", TimeNotSynchronized), (\"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203\", TimeNotSynchronized), (\"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec\", TimeNotSynchronized), (\"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957\", TimeNotSynchronized), (\"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284\", TimeNotSynchronized), (\"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\", TimeNotSynchronized), (\"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1\", TimeNotSynchronized)]","latency_us":515402,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:51.034766572Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"278.688335ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5a03f9ca-d7b8-4beb-bc7b-6fc87f940456\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 17:45:51 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5a03f9ca-d7b8-4beb-bc7b-6fc87f940456\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 8ae14f97-bd01-42ad-87d5-23eae9194521, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: bc60810c-eece-4cbd-a0f8-d04271924b7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bf3fc707-b452-4d30-a92b-e56b4d4122ee, kind: Service { id: bc60810c-eece-4cbd-a0f8-d04271924b7c }, name: Name(\"ntp-bc60810c-eece-4cbd-a0f8-d04271924b7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 196])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: c168b27b-f467-4350-a348-4006ce85a213, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 15be01e5-1765-45f0-b521-da99b395bdb3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: d8a1310d-09c1-4844-91b7-509f93b30cfd, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 59731bd7-ffc9-493f-a33c-a1cf5ba2df30, kind: Service { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1 }, name: Name(\"external-dns-1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 32])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 09fdc6fd-8297-436e-903d-b18710599203, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 005b2740-84d5-473e-ae59-aa79fd2fe5b6, kind: Service { id: 09fdc6fd-8297-436e-903d-b18710599203 }, name: Name(\"external-dns-09fdc6fd-8297-436e-903d-b18710599203\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: c6edf68c-f558-4f51-9aed-6eb91fb3ad51, kind: Service { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52 }, name: Name(\"nexus-87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 117])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6088410e-a8fc-4d33-b593-6385e863291f, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e6898038-bf11-41c8-b971-6ff3eed6d38b, kind: Service { id: 6088410e-a8fc-4d33-b593-6385e863291f }, name: Name(\"nexus-6088410e-a8fc-4d33-b593-6385e863291f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 27])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 742e0330-77de-4723-a145-74751d41c73b, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: ffe6d41d-1349-42b2-a887-49d734eef422, kind: Service { id: 742e0330-77de-4723-a145-74751d41c73b }, name: Name(\"nexus-742e0330-77de-4723-a145-74751d41c73b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8d644923-5967-4351-a1ad-fe8a938f33e1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 740bd3e0-b30c-4033-968a-5451c8d5adcd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3286a4ca-fffd-4ed0-9c21-d528e1d23603, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b1925c8-3ae5-4372-a3de-213551574fe1, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 663bd1f7-7a29-484c-bfce-d312646b1660, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: c6d325c3-cddf-4528-8ba6-a083f6511284, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External } } } }, OmicronZoneConfig { id: c30b7bc1-fd10-4942-8055-ada48d2e3957, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External } } } }, OmicronZoneConfig { id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:51.314344386Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:51.315264979Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:51.31527253Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:51.337988461Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:51.338044102Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:51.359395007Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:51.359440419Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:51.380560648Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:51.38061341Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:51.399281175Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:51.399332686Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:51.418762222Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:51.418815513Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:51.44019497Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:51.440250851Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:51.460022065Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:51.460071007Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:51.482711956Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:51.482765457Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:51.503361993Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:51.503416985Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:51.524264757Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:51.524307938Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:51.543856207Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:51.543904068Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:51.563908878Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:51.56397708Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:51.58474066Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:51.584786251Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:51.604803092Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:51.604850553Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:51.62583143Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:51.625875571Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:51.643941951Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:51.643989242Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:51.663863289Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:51.66391174Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:51.683642883Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:51.683693215Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:51.704107216Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:51.704161457Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:51.72423563Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/301424fa-c483-4ce9-92fb-3c8ee10e6f9f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:51.724298621Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/20dccfdd-e68a-468d-8629-7f7e0834aa1c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:51.724521697Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:51.724662651Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:51.823376709Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:51.823446421Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:51.823464331Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:51.823470621Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:51.823477922Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:51.823483402Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:51.823488562Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:51.823495502Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:51.823500842Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:51.823506212Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:51.823513013Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:51.823518823Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:51.823524203Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:51.823530183Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:51.823535543Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:51.823539363Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:51.823543493Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:51.823562664Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:51.823568614Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:51.823630056Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"991bf31d-0c0a-4434-a880-b81a79a04cf4","remote_addr":"[fd00:1122:3344:101::1]:37752","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b\", TimeNotSynchronized), (\"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7\", TimeNotSynchronized), (\"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\", TimeNotSynchronized), (\"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1\", TimeNotSynchronized), (\"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660\", TimeNotSynchronized), (\"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17\", TimeNotSynchronized), (\"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\", TimeNotSynchronized), (\"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603\", TimeNotSynchronized), (\"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b\", TimeNotSynchronized), (\"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9\", TimeNotSynchronized), (\"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb\", TimeNotSynchronized), (\"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957\", TimeNotSynchronized), (\"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1\", TimeNotSynchronized), (\"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1\", TimeNotSynchronized), (\"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284\", TimeNotSynchronized), (\"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec\", TimeNotSynchronized), (\"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd\", TimeNotSynchronized), (\"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203\", TimeNotSynchronized), (\"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f\", TimeNotSynchronized)]","latency_us":509135,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:51.823888172Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"442.383338ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"991bf31d-0c0a-4434-a880-b81a79a04cf4\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 17:45:51 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"991bf31d-0c0a-4434-a880-b81a79a04cf4\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 8ae14f97-bd01-42ad-87d5-23eae9194521, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: bc60810c-eece-4cbd-a0f8-d04271924b7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bf3fc707-b452-4d30-a92b-e56b4d4122ee, kind: Service { id: bc60810c-eece-4cbd-a0f8-d04271924b7c }, name: Name(\"ntp-bc60810c-eece-4cbd-a0f8-d04271924b7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 196])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: c168b27b-f467-4350-a348-4006ce85a213, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 15be01e5-1765-45f0-b521-da99b395bdb3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: d8a1310d-09c1-4844-91b7-509f93b30cfd, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 59731bd7-ffc9-493f-a33c-a1cf5ba2df30, kind: Service { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1 }, name: Name(\"external-dns-1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 32])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 09fdc6fd-8297-436e-903d-b18710599203, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 005b2740-84d5-473e-ae59-aa79fd2fe5b6, kind: Service { id: 09fdc6fd-8297-436e-903d-b18710599203 }, name: Name(\"external-dns-09fdc6fd-8297-436e-903d-b18710599203\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: c6edf68c-f558-4f51-9aed-6eb91fb3ad51, kind: Service { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52 }, name: Name(\"nexus-87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 117])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6088410e-a8fc-4d33-b593-6385e863291f, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e6898038-bf11-41c8-b971-6ff3eed6d38b, kind: Service { id: 6088410e-a8fc-4d33-b593-6385e863291f }, name: Name(\"nexus-6088410e-a8fc-4d33-b593-6385e863291f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 27])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 742e0330-77de-4723-a145-74751d41c73b, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: ffe6d41d-1349-42b2-a887-49d734eef422, kind: Service { id: 742e0330-77de-4723-a145-74751d41c73b }, name: Name(\"nexus-742e0330-77de-4723-a145-74751d41c73b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8d644923-5967-4351-a1ad-fe8a938f33e1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 740bd3e0-b30c-4033-968a-5451c8d5adcd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3286a4ca-fffd-4ed0-9c21-d528e1d23603, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b1925c8-3ae5-4372-a3de-213551574fe1, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 663bd1f7-7a29-484c-bfce-d312646b1660, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: c6d325c3-cddf-4528-8ba6-a083f6511284, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External } } } }, OmicronZoneConfig { id: c30b7bc1-fd10-4942-8055-ada48d2e3957, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External } } } }, OmicronZoneConfig { id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:52.268238373Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:52.268594862Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:52.268604842Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:52.29273752Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:52.292798002Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:52.316021126Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:52.316068617Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:52.33538021Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:52.335430461Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:52.353532382Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:52.353586544Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:52.372008473Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:52.372060044Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:52.394045526Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:52.394091767Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:52.415102754Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:52.415150015Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:52.435662289Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:52.43570946Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:52.455530476Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:52.455578527Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:52.476419089Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:52.476473441Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:52.497112548Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:52.497155879Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:52.517304243Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:52.517347264Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:52.538002862Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:52.538056303Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:52.559446489Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:52.559493561Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:52.578984008Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:52.579030269Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:52.599547573Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:52.599606274Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:52.621245627Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:52.621288468Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:52.642351686Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:52.642408818Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:52.662333416Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:52.662378497Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:52.682892151Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/301424fa-c483-4ce9-92fb-3c8ee10e6f9f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:52.682947713Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/20dccfdd-e68a-468d-8629-7f7e0834aa1c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:52.683143168Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:52.683269141Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:52.782736849Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:52.782801031Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:52.782809491Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:52.782815711Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:52.782821621Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:52.782829251Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:52.782846232Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:52.782853832Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:52.782859522Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:52.782864832Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:52.782871732Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:52.782877693Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:52.782883082Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:52.782888403Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:52.782895733Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:52.782901463Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:52.782907413Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:52.782931054Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:52.782940904Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:52.783088078Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"eea56224-6a5a-49e3-a37a-cdb7c0f1304d","remote_addr":"[fd00:1122:3344:101::1]:37752","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203\", TimeNotSynchronized), (\"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\", TimeNotSynchronized), (\"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec\", TimeNotSynchronized), (\"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7\", TimeNotSynchronized), (\"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b\", TimeNotSynchronized), (\"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb\", TimeNotSynchronized), (\"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17\", TimeNotSynchronized), (\"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f\", TimeNotSynchronized), (\"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957\", TimeNotSynchronized), (\"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284\", TimeNotSynchronized), (\"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b\", TimeNotSynchronized), (\"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603\", TimeNotSynchronized), (\"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1\", TimeNotSynchronized), (\"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd\", TimeNotSynchronized), (\"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1\", TimeNotSynchronized), (\"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660\", TimeNotSynchronized), (\"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1\", TimeNotSynchronized), (\"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9\", TimeNotSynchronized), (\"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\", TimeNotSynchronized)]","latency_us":514708,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:52.783351745Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"305.842624ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"eea56224-6a5a-49e3-a37a-cdb7c0f1304d\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 17:45:52 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"eea56224-6a5a-49e3-a37a-cdb7c0f1304d\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 8ae14f97-bd01-42ad-87d5-23eae9194521, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: bc60810c-eece-4cbd-a0f8-d04271924b7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bf3fc707-b452-4d30-a92b-e56b4d4122ee, kind: Service { id: bc60810c-eece-4cbd-a0f8-d04271924b7c }, name: Name(\"ntp-bc60810c-eece-4cbd-a0f8-d04271924b7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 196])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: c168b27b-f467-4350-a348-4006ce85a213, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 15be01e5-1765-45f0-b521-da99b395bdb3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: d8a1310d-09c1-4844-91b7-509f93b30cfd, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 59731bd7-ffc9-493f-a33c-a1cf5ba2df30, kind: Service { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1 }, name: Name(\"external-dns-1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 32])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 09fdc6fd-8297-436e-903d-b18710599203, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 005b2740-84d5-473e-ae59-aa79fd2fe5b6, kind: Service { id: 09fdc6fd-8297-436e-903d-b18710599203 }, name: Name(\"external-dns-09fdc6fd-8297-436e-903d-b18710599203\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: c6edf68c-f558-4f51-9aed-6eb91fb3ad51, kind: Service { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52 }, name: Name(\"nexus-87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 117])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6088410e-a8fc-4d33-b593-6385e863291f, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e6898038-bf11-41c8-b971-6ff3eed6d38b, kind: Service { id: 6088410e-a8fc-4d33-b593-6385e863291f }, name: Name(\"nexus-6088410e-a8fc-4d33-b593-6385e863291f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 27])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 742e0330-77de-4723-a145-74751d41c73b, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: ffe6d41d-1349-42b2-a887-49d734eef422, kind: Service { id: 742e0330-77de-4723-a145-74751d41c73b }, name: Name(\"nexus-742e0330-77de-4723-a145-74751d41c73b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8d644923-5967-4351-a1ad-fe8a938f33e1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 740bd3e0-b30c-4033-968a-5451c8d5adcd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3286a4ca-fffd-4ed0-9c21-d528e1d23603, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b1925c8-3ae5-4372-a3de-213551574fe1, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 663bd1f7-7a29-484c-bfce-d312646b1660, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: c6d325c3-cddf-4528-8ba6-a083f6511284, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External } } } }, OmicronZoneConfig { id: c30b7bc1-fd10-4942-8055-ada48d2e3957, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External } } } }, OmicronZoneConfig { id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.090854275Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.091201094Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.091251755Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.115038714Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.115091425Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.135573638Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.13562253Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.153791632Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.153840173Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.170836276Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.170886707Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.190011434Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.190062756Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.209393989Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.20944467Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.229579864Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.229634756Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.247962972Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.248005543Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.265303993Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.265346705Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.285408016Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.285463788Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.306813524Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.306858465Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.326967268Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.327020759Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.34474615Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.344789741Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.364277948Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.364318829Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.381652571Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.381695832Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.402267687Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.402314048Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.424937227Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.424984058Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.445577764Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.445621545Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.465302087Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.465362098Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.485234475Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/301424fa-c483-4ce9-92fb-3c8ee10e6f9f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.485291607Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/20dccfdd-e68a-468d-8629-7f7e0834aa1c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.485509883Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.485637226Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:53.583345968Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:53.5834056Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:53.58341393Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:53.58342001Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:53.58342404Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:53.58342958Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:53.58343526Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:53.583440581Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:53.583446021Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:53.583451731Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:53.583469721Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:53.583475682Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:53.583482702Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:53.583488462Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:53.583493872Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:53.583500732Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:53.583504892Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:53.583510102Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:53.583516772Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.583624035Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"3b5b5885-01d3-45f6-abe4-784a6e1231c1","remote_addr":"[fd00:1122:3344:101::1]:37752","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9\", TimeNotSynchronized), (\"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb\", TimeNotSynchronized), (\"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17\", TimeNotSynchronized), (\"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284\", TimeNotSynchronized), (\"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\", TimeNotSynchronized), (\"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203\", TimeNotSynchronized), (\"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7\", TimeNotSynchronized), (\"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957\", TimeNotSynchronized), (\"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f\", TimeNotSynchronized), (\"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b\", TimeNotSynchronized), (\"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b\", TimeNotSynchronized), (\"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\", TimeNotSynchronized), (\"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1\", TimeNotSynchronized), (\"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd\", TimeNotSynchronized), (\"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660\", TimeNotSynchronized), (\"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603\", TimeNotSynchronized), (\"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec\", TimeNotSynchronized), (\"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1\", TimeNotSynchronized), (\"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1\", TimeNotSynchronized)]","latency_us":492626,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:53.583870162Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"357.028805ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3b5b5885-01d3-45f6-abe4-784a6e1231c1\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 17:45:53 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3b5b5885-01d3-45f6-abe4-784a6e1231c1\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 8ae14f97-bd01-42ad-87d5-23eae9194521, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: bc60810c-eece-4cbd-a0f8-d04271924b7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bf3fc707-b452-4d30-a92b-e56b4d4122ee, kind: Service { id: bc60810c-eece-4cbd-a0f8-d04271924b7c }, name: Name(\"ntp-bc60810c-eece-4cbd-a0f8-d04271924b7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 196])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: c168b27b-f467-4350-a348-4006ce85a213, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 15be01e5-1765-45f0-b521-da99b395bdb3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: d8a1310d-09c1-4844-91b7-509f93b30cfd, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 59731bd7-ffc9-493f-a33c-a1cf5ba2df30, kind: Service { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1 }, name: Name(\"external-dns-1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 32])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 09fdc6fd-8297-436e-903d-b18710599203, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 005b2740-84d5-473e-ae59-aa79fd2fe5b6, kind: Service { id: 09fdc6fd-8297-436e-903d-b18710599203 }, name: Name(\"external-dns-09fdc6fd-8297-436e-903d-b18710599203\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: c6edf68c-f558-4f51-9aed-6eb91fb3ad51, kind: Service { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52 }, name: Name(\"nexus-87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 117])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6088410e-a8fc-4d33-b593-6385e863291f, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e6898038-bf11-41c8-b971-6ff3eed6d38b, kind: Service { id: 6088410e-a8fc-4d33-b593-6385e863291f }, name: Name(\"nexus-6088410e-a8fc-4d33-b593-6385e863291f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 27])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 742e0330-77de-4723-a145-74751d41c73b, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: ffe6d41d-1349-42b2-a887-49d734eef422, kind: Service { id: 742e0330-77de-4723-a145-74751d41c73b }, name: Name(\"nexus-742e0330-77de-4723-a145-74751d41c73b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8d644923-5967-4351-a1ad-fe8a938f33e1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 740bd3e0-b30c-4033-968a-5451c8d5adcd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3286a4ca-fffd-4ed0-9c21-d528e1d23603, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b1925c8-3ae5-4372-a3de-213551574fe1, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 663bd1f7-7a29-484c-bfce-d312646b1660, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: c6d325c3-cddf-4528-8ba6-a083f6511284, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External } } } }, OmicronZoneConfig { id: c30b7bc1-fd10-4942-8055-ada48d2e3957, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External } } } }, OmicronZoneConfig { id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.941383113Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.941743032Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.941798444Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.965065599Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.965118671Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.986844896Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:53.986894167Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:54.007647367Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:54.007695528Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:54.032006201Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:54.032054742Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:54.054209249Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:54.054250819Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:54.077994587Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:54.078054559Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:54.099762443Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:54.099810325Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:54.119943629Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:54.1199917Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:54.138279196Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:54.138326277Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:54.16037048Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:54.160424072Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:54.180970576Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:54.181027818Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:54.200481674Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:54.200522605Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:54.221294216Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:54.221339607Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:54.243054272Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:54.243095423Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:54.262518068Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:54.262563219Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:54.282624551Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:54.282669462Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:54.303804242Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:54.303859384Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:54.325921438Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:54.325968869Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:54.349609924Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:54.349656645Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:54.370360684Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/301424fa-c483-4ce9-92fb-3c8ee10e6f9f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:54.370417705Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/20dccfdd-e68a-468d-8629-7f7e0834aa1c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:54.370631421Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:54.370762794Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:54.47208154Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:54.472141612Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:54.472150182Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:54.472157352Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:54.472162952Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:54.472170113Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:54.472173863Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:54.472180613Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:54.472186063Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:54.472191583Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:54.472197193Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:54.472202644Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:54.472208124Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:54.472225354Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:54.472229514Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:54.472234884Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:54.472238744Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:54.472242345Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:54.472246085Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:54.472335227Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"7497e40f-6825-4867-a306-87661587dc61","remote_addr":"[fd00:1122:3344:101::1]:37752","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec\", TimeNotSynchronized), (\"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7\", TimeNotSynchronized), (\"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17\", TimeNotSynchronized), (\"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9\", TimeNotSynchronized), (\"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\", TimeNotSynchronized), (\"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660\", TimeNotSynchronized), (\"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\", TimeNotSynchronized), (\"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f\", TimeNotSynchronized), (\"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603\", TimeNotSynchronized), (\"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd\", TimeNotSynchronized), (\"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1\", TimeNotSynchronized), (\"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1\", TimeNotSynchronized), (\"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1\", TimeNotSynchronized), (\"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b\", TimeNotSynchronized), (\"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b\", TimeNotSynchronized), (\"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284\", TimeNotSynchronized), (\"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb\", TimeNotSynchronized), (\"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957\", TimeNotSynchronized), (\"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203\", TimeNotSynchronized)]","latency_us":530795,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:54.472572963Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"709.831138ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7497e40f-6825-4867-a306-87661587dc61\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 17:45:54 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7497e40f-6825-4867-a306-87661587dc61\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 8ae14f97-bd01-42ad-87d5-23eae9194521, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: bc60810c-eece-4cbd-a0f8-d04271924b7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bf3fc707-b452-4d30-a92b-e56b4d4122ee, kind: Service { id: bc60810c-eece-4cbd-a0f8-d04271924b7c }, name: Name(\"ntp-bc60810c-eece-4cbd-a0f8-d04271924b7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 196])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: c168b27b-f467-4350-a348-4006ce85a213, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 15be01e5-1765-45f0-b521-da99b395bdb3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: d8a1310d-09c1-4844-91b7-509f93b30cfd, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 59731bd7-ffc9-493f-a33c-a1cf5ba2df30, kind: Service { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1 }, name: Name(\"external-dns-1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 32])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 09fdc6fd-8297-436e-903d-b18710599203, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 005b2740-84d5-473e-ae59-aa79fd2fe5b6, kind: Service { id: 09fdc6fd-8297-436e-903d-b18710599203 }, name: Name(\"external-dns-09fdc6fd-8297-436e-903d-b18710599203\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: c6edf68c-f558-4f51-9aed-6eb91fb3ad51, kind: Service { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52 }, name: Name(\"nexus-87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 117])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6088410e-a8fc-4d33-b593-6385e863291f, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e6898038-bf11-41c8-b971-6ff3eed6d38b, kind: Service { id: 6088410e-a8fc-4d33-b593-6385e863291f }, name: Name(\"nexus-6088410e-a8fc-4d33-b593-6385e863291f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 27])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 742e0330-77de-4723-a145-74751d41c73b, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: ffe6d41d-1349-42b2-a887-49d734eef422, kind: Service { id: 742e0330-77de-4723-a145-74751d41c73b }, name: Name(\"nexus-742e0330-77de-4723-a145-74751d41c73b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8d644923-5967-4351-a1ad-fe8a938f33e1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 740bd3e0-b30c-4033-968a-5451c8d5adcd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3286a4ca-fffd-4ed0-9c21-d528e1d23603, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b1925c8-3ae5-4372-a3de-213551574fe1, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 663bd1f7-7a29-484c-bfce-d312646b1660, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: c6d325c3-cddf-4528-8ba6-a083f6511284, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External } } } }, OmicronZoneConfig { id: c30b7bc1-fd10-4942-8055-ada48d2e3957, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External } } } }, OmicronZoneConfig { id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:55.184041683Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:55.184483375Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:55.184550927Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:55.20888129Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:55.208934601Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:55.230094831Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:55.230138003Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:55.250300557Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:55.250352439Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:55.270447892Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:55.270496283Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:55.289353413Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:55.289399465Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:55.309789495Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:55.309832396Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:55.328281646Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:55.328324887Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:55.348065921Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:55.348108922Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:55.36916641Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:55.369209091Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:55.38914861Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:55.389197731Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:55.408502213Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:55.408548314Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:55.427683912Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:55.427743254Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:55.448549285Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:55.448593456Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:55.469858869Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:55.469902261Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:55.489519941Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:55.489564682Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:55.509991524Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:55.510034715Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:55.530959659Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:55.53100163Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:55.550676622Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:55.550724124Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:55.57060101Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:55.570650502Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:55.589671737Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/301424fa-c483-4ce9-92fb-3c8ee10e6f9f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:55.589725528Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/20dccfdd-e68a-468d-8629-7f7e0834aa1c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:55.589941764Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:55.590061657Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:55.688112508Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:55.68817762Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:55.68820421Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:55.68820883Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:55.68821423Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:55.688219681Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:55.688226741Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:55.688232111Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:55.688237451Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:55.688243001Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:55.688250502Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:55.688255902Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:55.688261642Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:55.688267182Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:55.688272662Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:55.688279642Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:55.688285022Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:55.688293943Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:55.688297983Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:55.688384525Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"38706219-c761-4414-9ba6-87b319228af6","remote_addr":"[fd00:1122:3344:101::1]:37752","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203\", TimeNotSynchronized), (\"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\", TimeNotSynchronized), (\"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd\", TimeNotSynchronized), (\"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660\", TimeNotSynchronized), (\"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec\", TimeNotSynchronized), (\"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9\", TimeNotSynchronized), (\"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7\", TimeNotSynchronized), (\"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb\", TimeNotSynchronized), (\"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1\", TimeNotSynchronized), (\"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957\", TimeNotSynchronized), (\"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\", TimeNotSynchronized), (\"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b\", TimeNotSynchronized), (\"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1\", TimeNotSynchronized), (\"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b\", TimeNotSynchronized), (\"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1\", TimeNotSynchronized), (\"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284\", TimeNotSynchronized), (\"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603\", TimeNotSynchronized), (\"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17\", TimeNotSynchronized), (\"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f\", TimeNotSynchronized)]","latency_us":504171,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:55.688629261Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"1.191411395s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"38706219-c761-4414-9ba6-87b319228af6\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 17:45:55 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"38706219-c761-4414-9ba6-87b319228af6\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 8ae14f97-bd01-42ad-87d5-23eae9194521, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: bc60810c-eece-4cbd-a0f8-d04271924b7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bf3fc707-b452-4d30-a92b-e56b4d4122ee, kind: Service { id: bc60810c-eece-4cbd-a0f8-d04271924b7c }, name: Name(\"ntp-bc60810c-eece-4cbd-a0f8-d04271924b7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 196])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: c168b27b-f467-4350-a348-4006ce85a213, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 15be01e5-1765-45f0-b521-da99b395bdb3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: d8a1310d-09c1-4844-91b7-509f93b30cfd, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 59731bd7-ffc9-493f-a33c-a1cf5ba2df30, kind: Service { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1 }, name: Name(\"external-dns-1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 32])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 09fdc6fd-8297-436e-903d-b18710599203, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 005b2740-84d5-473e-ae59-aa79fd2fe5b6, kind: Service { id: 09fdc6fd-8297-436e-903d-b18710599203 }, name: Name(\"external-dns-09fdc6fd-8297-436e-903d-b18710599203\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: c6edf68c-f558-4f51-9aed-6eb91fb3ad51, kind: Service { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52 }, name: Name(\"nexus-87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 117])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6088410e-a8fc-4d33-b593-6385e863291f, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e6898038-bf11-41c8-b971-6ff3eed6d38b, kind: Service { id: 6088410e-a8fc-4d33-b593-6385e863291f }, name: Name(\"nexus-6088410e-a8fc-4d33-b593-6385e863291f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 27])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 742e0330-77de-4723-a145-74751d41c73b, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: ffe6d41d-1349-42b2-a887-49d734eef422, kind: Service { id: 742e0330-77de-4723-a145-74751d41c73b }, name: Name(\"nexus-742e0330-77de-4723-a145-74751d41c73b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8d644923-5967-4351-a1ad-fe8a938f33e1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 740bd3e0-b30c-4033-968a-5451c8d5adcd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3286a4ca-fffd-4ed0-9c21-d528e1d23603, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b1925c8-3ae5-4372-a3de-213551574fe1, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 663bd1f7-7a29-484c-bfce-d312646b1660, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: c6d325c3-cddf-4528-8ba6-a083f6511284, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External } } } }, OmicronZoneConfig { id: c30b7bc1-fd10-4942-8055-ada48d2e3957, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External } } } }, OmicronZoneConfig { id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:56.880630934Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:56.880964342Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:56.880985263Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:56.904192987Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:56.904235458Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:56.924525246Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:56.924574277Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:56.945528232Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:56.945576043Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:56.964325201Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:56.964383393Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:56.982200656Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:56.982244708Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:57.002659638Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:57.002705959Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:57.023568452Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:57.023627233Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:57.045203825Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:57.045260126Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:57.065627516Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:57.065672357Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:57.087480525Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:57.087527106Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:57.108976304Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:57.109021685Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:57.131300094Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:57.131352976Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:57.15150612Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:57.151565802Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:57.170754141Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:57.170800122Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:57.191583683Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:57.191625814Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:57.211315956Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:57.211358637Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:57.234954211Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:57.235002463Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:57.25645255Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:57.256505712Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:57.277426656Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:57.277471157Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:57.298461414Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/301424fa-c483-4ce9-92fb-3c8ee10e6f9f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:57.298517985Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/20dccfdd-e68a-468d-8629-7f7e0834aa1c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:57.29873219Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:57.298881965Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:57.398469045Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:57.398539287Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:57.398547858Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:57.398553448Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:57.398560228Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:57.398565488Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:57.398589719Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:57.398593719Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:57.398597379Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:57.398602389Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:57.398607759Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:57.398614039Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:57.398621669Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:57.39862905Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:57.39863437Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:57.398638Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:57.39864217Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:57.39864745Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:57.39865226Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:57.398683591Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"943b5daa-f42e-4315-ba70-8dfeeb084b49","remote_addr":"[fd00:1122:3344:101::1]:37752","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957\", TimeNotSynchronized), (\"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\", TimeNotSynchronized), (\"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1\", TimeNotSynchronized), (\"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b\", TimeNotSynchronized), (\"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17\", TimeNotSynchronized), (\"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\", TimeNotSynchronized), (\"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9\", TimeNotSynchronized), (\"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284\", TimeNotSynchronized), (\"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b\", TimeNotSynchronized), (\"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660\", TimeNotSynchronized), (\"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203\", TimeNotSynchronized), (\"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f\", TimeNotSynchronized), (\"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603\", TimeNotSynchronized), (\"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1\", TimeNotSynchronized), (\"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec\", TimeNotSynchronized), (\"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd\", TimeNotSynchronized), (\"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1\", TimeNotSynchronized), (\"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7\", TimeNotSynchronized), (\"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb\", TimeNotSynchronized)]","latency_us":517928,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:57.398909077Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"666.726847ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"943b5daa-f42e-4315-ba70-8dfeeb084b49\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 17:45:56 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"943b5daa-f42e-4315-ba70-8dfeeb084b49\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 8ae14f97-bd01-42ad-87d5-23eae9194521, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: bc60810c-eece-4cbd-a0f8-d04271924b7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bf3fc707-b452-4d30-a92b-e56b4d4122ee, kind: Service { id: bc60810c-eece-4cbd-a0f8-d04271924b7c }, name: Name(\"ntp-bc60810c-eece-4cbd-a0f8-d04271924b7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 196])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: c168b27b-f467-4350-a348-4006ce85a213, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 15be01e5-1765-45f0-b521-da99b395bdb3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: d8a1310d-09c1-4844-91b7-509f93b30cfd, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 59731bd7-ffc9-493f-a33c-a1cf5ba2df30, kind: Service { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1 }, name: Name(\"external-dns-1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 32])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 09fdc6fd-8297-436e-903d-b18710599203, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 005b2740-84d5-473e-ae59-aa79fd2fe5b6, kind: Service { id: 09fdc6fd-8297-436e-903d-b18710599203 }, name: Name(\"external-dns-09fdc6fd-8297-436e-903d-b18710599203\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: c6edf68c-f558-4f51-9aed-6eb91fb3ad51, kind: Service { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52 }, name: Name(\"nexus-87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 117])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6088410e-a8fc-4d33-b593-6385e863291f, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e6898038-bf11-41c8-b971-6ff3eed6d38b, kind: Service { id: 6088410e-a8fc-4d33-b593-6385e863291f }, name: Name(\"nexus-6088410e-a8fc-4d33-b593-6385e863291f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 27])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 742e0330-77de-4723-a145-74751d41c73b, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: ffe6d41d-1349-42b2-a887-49d734eef422, kind: Service { id: 742e0330-77de-4723-a145-74751d41c73b }, name: Name(\"nexus-742e0330-77de-4723-a145-74751d41c73b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8d644923-5967-4351-a1ad-fe8a938f33e1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 740bd3e0-b30c-4033-968a-5451c8d5adcd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3286a4ca-fffd-4ed0-9c21-d528e1d23603, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b1925c8-3ae5-4372-a3de-213551574fe1, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 663bd1f7-7a29-484c-bfce-d312646b1660, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: c6d325c3-cddf-4528-8ba6-a083f6511284, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External } } } }, OmicronZoneConfig { id: c30b7bc1-fd10-4942-8055-ada48d2e3957, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External } } } }, OmicronZoneConfig { id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:58.066581238Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:58.067539432Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:58.067595414Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:58.091498706Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:58.091551007Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:58.112531593Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:58.112575424Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:58.132466432Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:58.132514073Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:58.153800737Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:58.153846348Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:58.172705008Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:58.17276143Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:58.190244105Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:58.190284846Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:58.210000189Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:58.21004072Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:58.230261886Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:58.230300937Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:58.249816175Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:58.249862186Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:58.269337693Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:58.269382544Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:58.290665377Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:58.290717859Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:58.312012113Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:58.312063384Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:58.332238899Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:58.3322797Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:58.351822459Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:58.35186678Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:58.371173982Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:58.371214283Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:58.392276561Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:58.392314752Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:58.413043601Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:58.413084042Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:58.433471503Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:58.433511174Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:58.454280944Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:58.454330096Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:58.474884581Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/301424fa-c483-4ce9-92fb-3c8ee10e6f9f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:58.474934162Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/20dccfdd-e68a-468d-8629-7f7e0834aa1c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:58.475127117Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:58.4752483Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:58.573872706Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:58.573939887Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:58.573946538Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:58.573950498Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:58.573955758Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:58.573961368Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:58.573966878Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:58.573972268Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:58.573977788Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:58.573981609Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:58.574010349Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:58.5740164Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:58.57402184Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:58.5740272Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:58.57403259Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:58.57403775Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:58.57404315Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:58.574049921Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:58.574055261Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:45:58.574153963Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"335349a7-9402-43a6-a6de-63d3ad659fae","remote_addr":"[fd00:1122:3344:101::1]:37752","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec\", TimeNotSynchronized), (\"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1\", TimeNotSynchronized), (\"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb\", TimeNotSynchronized), (\"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957\", TimeNotSynchronized), (\"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17\", TimeNotSynchronized), (\"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284\", TimeNotSynchronized), (\"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b\", TimeNotSynchronized), (\"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203\", TimeNotSynchronized), (\"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1\", TimeNotSynchronized), (\"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660\", TimeNotSynchronized), (\"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\", TimeNotSynchronized), (\"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9\", TimeNotSynchronized), (\"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1\", TimeNotSynchronized), (\"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603\", TimeNotSynchronized), (\"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7\", TimeNotSynchronized), (\"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\", TimeNotSynchronized), (\"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd\", TimeNotSynchronized), (\"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f\", TimeNotSynchronized), (\"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b\", TimeNotSynchronized)]","latency_us":507419,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:58.574400519Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"1.527556121s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"335349a7-9402-43a6-a6de-63d3ad659fae\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 17:45:58 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"335349a7-9402-43a6-a6de-63d3ad659fae\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:45:58.941791478Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"2.283909868s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 8ae14f97-bd01-42ad-87d5-23eae9194521, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: bc60810c-eece-4cbd-a0f8-d04271924b7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bf3fc707-b452-4d30-a92b-e56b4d4122ee, kind: Service { id: bc60810c-eece-4cbd-a0f8-d04271924b7c }, name: Name(\"ntp-bc60810c-eece-4cbd-a0f8-d04271924b7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 196])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: c168b27b-f467-4350-a348-4006ce85a213, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 15be01e5-1765-45f0-b521-da99b395bdb3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: d8a1310d-09c1-4844-91b7-509f93b30cfd, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 59731bd7-ffc9-493f-a33c-a1cf5ba2df30, kind: Service { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1 }, name: Name(\"external-dns-1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 32])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 09fdc6fd-8297-436e-903d-b18710599203, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 005b2740-84d5-473e-ae59-aa79fd2fe5b6, kind: Service { id: 09fdc6fd-8297-436e-903d-b18710599203 }, name: Name(\"external-dns-09fdc6fd-8297-436e-903d-b18710599203\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: c6edf68c-f558-4f51-9aed-6eb91fb3ad51, kind: Service { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52 }, name: Name(\"nexus-87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 117])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6088410e-a8fc-4d33-b593-6385e863291f, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e6898038-bf11-41c8-b971-6ff3eed6d38b, kind: Service { id: 6088410e-a8fc-4d33-b593-6385e863291f }, name: Name(\"nexus-6088410e-a8fc-4d33-b593-6385e863291f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 27])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 742e0330-77de-4723-a145-74751d41c73b, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: ffe6d41d-1349-42b2-a887-49d734eef422, kind: Service { id: 742e0330-77de-4723-a145-74751d41c73b }, name: Name(\"nexus-742e0330-77de-4723-a145-74751d41c73b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8d644923-5967-4351-a1ad-fe8a938f33e1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 740bd3e0-b30c-4033-968a-5451c8d5adcd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3286a4ca-fffd-4ed0-9c21-d528e1d23603, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b1925c8-3ae5-4372-a3de-213551574fe1, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 663bd1f7-7a29-484c-bfce-d312646b1660, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: c6d325c3-cddf-4528-8ba6-a083f6511284, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External } } } }, OmicronZoneConfig { id: c30b7bc1-fd10-4942-8055-ada48d2e3957, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External } } } }, OmicronZoneConfig { id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:00.102404123Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:00.102719231Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:00.102728751Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:00.125202706Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:00.125260567Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:00.146815998Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:00.14686102Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:00.166181012Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:00.166228173Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:00.187856036Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:00.187898887Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:00.207389614Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:00.207447536Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:00.228530774Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:00.228578955Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:00.247082857Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:00.247125108Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:00.265749973Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:00.265797184Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:00.286063021Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:00.286106912Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:00.305508477Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:00.305564829Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:00.325021365Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:00.325070936Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:00.346442222Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:00.346493313Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:00.366106774Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:00.366160185Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:00.386171746Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:00.386216257Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:00.407605413Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:00.407680215Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:00.428680842Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:00.428729313Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:00.450101849Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:00.4501461Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:00.469627797Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:00.469676078Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:00.491228719Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:00.49127758Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:00.511896187Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/301424fa-c483-4ce9-92fb-3c8ee10e6f9f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:00.511964099Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/20dccfdd-e68a-468d-8629-7f7e0834aa1c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:00.512178384Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:00.512334028Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:00.610694237Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:00.610753609Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:00.610761819Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:00.610771259Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:00.610778589Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:00.61078587Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:00.61079461Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:00.61080147Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:00.61080855Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:00.61081557Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:00.610824051Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:00.610851381Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:00.610858441Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:00.610865601Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:00.610874182Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:00.610881192Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:00.610886652Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:00.610892582Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:00.610900013Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:00.611012305Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"63ff7b13-f19f-42cc-bf9d-c0e1cc6fe09a","remote_addr":"[fd00:1122:3344:101::1]:37752","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1\", TimeNotSynchronized), (\"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1\", TimeNotSynchronized), (\"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b\", TimeNotSynchronized), (\"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1\", TimeNotSynchronized), (\"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284\", TimeNotSynchronized), (\"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7\", TimeNotSynchronized), (\"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957\", TimeNotSynchronized), (\"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd\", TimeNotSynchronized), (\"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\", TimeNotSynchronized), (\"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660\", TimeNotSynchronized), (\"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b\", TimeNotSynchronized), (\"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603\", TimeNotSynchronized), (\"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\", TimeNotSynchronized), (\"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203\", TimeNotSynchronized), (\"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f\", TimeNotSynchronized), (\"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb\", TimeNotSynchronized), (\"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17\", TimeNotSynchronized), (\"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec\", TimeNotSynchronized), (\"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9\", TimeNotSynchronized)]","latency_us":508470,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:00.611315543Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"1.227112834s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"63ff7b13-f19f-42cc-bf9d-c0e1cc6fe09a\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 17:46:00 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"63ff7b13-f19f-42cc-bf9d-c0e1cc6fe09a\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 8ae14f97-bd01-42ad-87d5-23eae9194521, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: bc60810c-eece-4cbd-a0f8-d04271924b7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bf3fc707-b452-4d30-a92b-e56b4d4122ee, kind: Service { id: bc60810c-eece-4cbd-a0f8-d04271924b7c }, name: Name(\"ntp-bc60810c-eece-4cbd-a0f8-d04271924b7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 196])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: c168b27b-f467-4350-a348-4006ce85a213, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 15be01e5-1765-45f0-b521-da99b395bdb3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: d8a1310d-09c1-4844-91b7-509f93b30cfd, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 59731bd7-ffc9-493f-a33c-a1cf5ba2df30, kind: Service { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1 }, name: Name(\"external-dns-1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 32])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 09fdc6fd-8297-436e-903d-b18710599203, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 005b2740-84d5-473e-ae59-aa79fd2fe5b6, kind: Service { id: 09fdc6fd-8297-436e-903d-b18710599203 }, name: Name(\"external-dns-09fdc6fd-8297-436e-903d-b18710599203\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: c6edf68c-f558-4f51-9aed-6eb91fb3ad51, kind: Service { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52 }, name: Name(\"nexus-87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 117])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6088410e-a8fc-4d33-b593-6385e863291f, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e6898038-bf11-41c8-b971-6ff3eed6d38b, kind: Service { id: 6088410e-a8fc-4d33-b593-6385e863291f }, name: Name(\"nexus-6088410e-a8fc-4d33-b593-6385e863291f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 27])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 742e0330-77de-4723-a145-74751d41c73b, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: ffe6d41d-1349-42b2-a887-49d734eef422, kind: Service { id: 742e0330-77de-4723-a145-74751d41c73b }, name: Name(\"nexus-742e0330-77de-4723-a145-74751d41c73b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8d644923-5967-4351-a1ad-fe8a938f33e1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 740bd3e0-b30c-4033-968a-5451c8d5adcd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3286a4ca-fffd-4ed0-9c21-d528e1d23603, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b1925c8-3ae5-4372-a3de-213551574fe1, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 663bd1f7-7a29-484c-bfce-d312646b1660, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: c6d325c3-cddf-4528-8ba6-a083f6511284, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External } } } }, OmicronZoneConfig { id: c30b7bc1-fd10-4942-8055-ada48d2e3957, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External } } } }, OmicronZoneConfig { id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:01.840242976Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:01.840622066Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:01.840631116Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:01.863569743Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:01.863614144Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:01.886367856Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:01.886419667Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:01.905933695Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:01.905975656Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:01.927320972Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:01.927376663Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:01.951034009Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:01.95108042Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:01.974258163Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:01.974308024Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:01.994860348Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:01.994908059Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:02.015722041Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:02.015761842Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:02.037879597Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:02.037923578Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:02.058855263Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:02.058899954Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:02.080518487Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:02.080557948Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:02.100657651Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:02.100699132Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:02.12099694Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:02.121037541Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:02.14137641Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:02.141429001Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:02.161518884Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:02.161556585Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:02.181659138Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:02.181717509Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:02.202332046Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:02.202371307Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:02.222804258Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:02.22284499Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:02.241655869Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:02.24169615Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:02.262394508Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/301424fa-c483-4ce9-92fb-3c8ee10e6f9f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:02.2624538Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/20dccfdd-e68a-468d-8629-7f7e0834aa1c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:02.262660175Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:02.262776248Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:02.365817109Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:02.365885251Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:02.365894491Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:02.365900621Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:02.365907712Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:02.365913402Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:02.365919262Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:02.365927102Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:02.365932702Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:02.365938472Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:02.365945582Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:02.365951243Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:02.365958473Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:02.365964303Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:02.365970013Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:02.365997974Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:02.366004064Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:02.366011214Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:02.366023675Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:02.366143988Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"a59f0591-4621-4c8e-90c6-30e4047d3c9a","remote_addr":"[fd00:1122:3344:101::1]:37752","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1\", TimeNotSynchronized), (\"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb\", TimeNotSynchronized), (\"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b\", TimeNotSynchronized), (\"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957\", TimeNotSynchronized), (\"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec\", TimeNotSynchronized), (\"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1\", TimeNotSynchronized), (\"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603\", TimeNotSynchronized), (\"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b\", TimeNotSynchronized), (\"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660\", TimeNotSynchronized), (\"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\", TimeNotSynchronized), (\"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\", TimeNotSynchronized), (\"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203\", TimeNotSynchronized), (\"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9\", TimeNotSynchronized), (\"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284\", TimeNotSynchronized), (\"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7\", TimeNotSynchronized), (\"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17\", TimeNotSynchronized), (\"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1\", TimeNotSynchronized), (\"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f\", TimeNotSynchronized), (\"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd\", TimeNotSynchronized)]","latency_us":525747,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:02.366315812Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"2.348055205s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a59f0591-4621-4c8e-90c6-30e4047d3c9a\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 17:46:01 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a59f0591-4621-4c8e-90c6-30e4047d3c9a\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 8ae14f97-bd01-42ad-87d5-23eae9194521, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: bc60810c-eece-4cbd-a0f8-d04271924b7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bf3fc707-b452-4d30-a92b-e56b4d4122ee, kind: Service { id: bc60810c-eece-4cbd-a0f8-d04271924b7c }, name: Name(\"ntp-bc60810c-eece-4cbd-a0f8-d04271924b7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 196])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: c168b27b-f467-4350-a348-4006ce85a213, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 15be01e5-1765-45f0-b521-da99b395bdb3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: d8a1310d-09c1-4844-91b7-509f93b30cfd, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 59731bd7-ffc9-493f-a33c-a1cf5ba2df30, kind: Service { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1 }, name: Name(\"external-dns-1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 32])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 09fdc6fd-8297-436e-903d-b18710599203, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 005b2740-84d5-473e-ae59-aa79fd2fe5b6, kind: Service { id: 09fdc6fd-8297-436e-903d-b18710599203 }, name: Name(\"external-dns-09fdc6fd-8297-436e-903d-b18710599203\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: c6edf68c-f558-4f51-9aed-6eb91fb3ad51, kind: Service { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52 }, name: Name(\"nexus-87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 117])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6088410e-a8fc-4d33-b593-6385e863291f, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e6898038-bf11-41c8-b971-6ff3eed6d38b, kind: Service { id: 6088410e-a8fc-4d33-b593-6385e863291f }, name: Name(\"nexus-6088410e-a8fc-4d33-b593-6385e863291f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 27])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 742e0330-77de-4723-a145-74751d41c73b, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: ffe6d41d-1349-42b2-a887-49d734eef422, kind: Service { id: 742e0330-77de-4723-a145-74751d41c73b }, name: Name(\"nexus-742e0330-77de-4723-a145-74751d41c73b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8d644923-5967-4351-a1ad-fe8a938f33e1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 740bd3e0-b30c-4033-968a-5451c8d5adcd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3286a4ca-fffd-4ed0-9c21-d528e1d23603, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b1925c8-3ae5-4372-a3de-213551574fe1, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 663bd1f7-7a29-484c-bfce-d312646b1660, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: c6d325c3-cddf-4528-8ba6-a083f6511284, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External } } } }, OmicronZoneConfig { id: c30b7bc1-fd10-4942-8055-ada48d2e3957, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External } } } }, OmicronZoneConfig { id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:04.71430874Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:04.71469367Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:04.714752352Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:04.73808639Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:04.738128692Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:04.757918948Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:04.757962889Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:04.778402322Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:04.778460254Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:04.799225205Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:04.799264277Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:04.818602371Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:04.818640652Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:04.83812969Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:04.838174692Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:04.858224175Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:04.858266816Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:04.877957129Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:04.878011581Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:04.896985606Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:04.897029367Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:04.916126105Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:04.916177507Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:04.934474314Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:04.934518705Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:04.952905155Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:04.952946686Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:04.972920727Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:04.972962288Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:04.995931817Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:04.995976168Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:05.017131892Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:05.017179771Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:05.036496375Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:05.036538196Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:05.05506927Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:05.055126551Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:05.074034965Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:05.074082226Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:05.094523589Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:05.094579421Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:05.117011176Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/301424fa-c483-4ce9-92fb-3c8ee10e6f9f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:05.117067427Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/20dccfdd-e68a-468d-8629-7f7e0834aa1c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:05.117296863Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:05.117447757Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:05.213967805Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:05.214031287Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:05.214039107Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:05.214046147Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:05.214063447Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:05.214070398Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:05.214078848Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:05.214085308Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:05.214091608Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:05.214099168Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:05.214105569Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:05.214113399Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:05.214119599Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:05.214125799Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:05.214133459Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:05.214139549Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:05.214144Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:05.21415001Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:05.21415442Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:05.214238362Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"9541333b-af85-434f-9d61-a73be126c9af","remote_addr":"[fd00:1122:3344:101::1]:37752","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9\", TimeNotSynchronized), (\"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b\", TimeNotSynchronized), (\"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\", TimeNotSynchronized), (\"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284\", TimeNotSynchronized), (\"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1\", TimeNotSynchronized), (\"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb\", TimeNotSynchronized), (\"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603\", TimeNotSynchronized), (\"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec\", TimeNotSynchronized), (\"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7\", TimeNotSynchronized), (\"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\", TimeNotSynchronized), (\"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203\", TimeNotSynchronized), (\"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f\", TimeNotSynchronized), (\"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd\", TimeNotSynchronized), (\"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1\", TimeNotSynchronized), (\"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957\", TimeNotSynchronized), (\"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660\", TimeNotSynchronized), (\"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17\", TimeNotSynchronized), (\"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b\", TimeNotSynchronized), (\"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1\", TimeNotSynchronized)]","latency_us":501134,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:05.214423587Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"2.128911756s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9541333b-af85-434f-9d61-a73be126c9af\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 17:46:04 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9541333b-af85-434f-9d61-a73be126c9af\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 8ae14f97-bd01-42ad-87d5-23eae9194521, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: bc60810c-eece-4cbd-a0f8-d04271924b7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bf3fc707-b452-4d30-a92b-e56b4d4122ee, kind: Service { id: bc60810c-eece-4cbd-a0f8-d04271924b7c }, name: Name(\"ntp-bc60810c-eece-4cbd-a0f8-d04271924b7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 196])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: c168b27b-f467-4350-a348-4006ce85a213, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 15be01e5-1765-45f0-b521-da99b395bdb3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: d8a1310d-09c1-4844-91b7-509f93b30cfd, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 59731bd7-ffc9-493f-a33c-a1cf5ba2df30, kind: Service { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1 }, name: Name(\"external-dns-1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 32])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 09fdc6fd-8297-436e-903d-b18710599203, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 005b2740-84d5-473e-ae59-aa79fd2fe5b6, kind: Service { id: 09fdc6fd-8297-436e-903d-b18710599203 }, name: Name(\"external-dns-09fdc6fd-8297-436e-903d-b18710599203\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: c6edf68c-f558-4f51-9aed-6eb91fb3ad51, kind: Service { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52 }, name: Name(\"nexus-87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 117])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6088410e-a8fc-4d33-b593-6385e863291f, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e6898038-bf11-41c8-b971-6ff3eed6d38b, kind: Service { id: 6088410e-a8fc-4d33-b593-6385e863291f }, name: Name(\"nexus-6088410e-a8fc-4d33-b593-6385e863291f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 27])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 742e0330-77de-4723-a145-74751d41c73b, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: ffe6d41d-1349-42b2-a887-49d734eef422, kind: Service { id: 742e0330-77de-4723-a145-74751d41c73b }, name: Name(\"nexus-742e0330-77de-4723-a145-74751d41c73b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8d644923-5967-4351-a1ad-fe8a938f33e1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 740bd3e0-b30c-4033-968a-5451c8d5adcd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3286a4ca-fffd-4ed0-9c21-d528e1d23603, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b1925c8-3ae5-4372-a3de-213551574fe1, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 663bd1f7-7a29-484c-bfce-d312646b1660, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: c6d325c3-cddf-4528-8ba6-a083f6511284, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External } } } }, OmicronZoneConfig { id: c30b7bc1-fd10-4942-8055-ada48d2e3957, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External } } } }, OmicronZoneConfig { id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:07.338340764Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:07.338688573Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:07.338707204Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:07.361266022Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:07.361314603Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:07.381723645Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:07.381767407Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:07.40105405Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:07.401100492Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:07.419589834Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:07.419633465Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:07.441478765Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:07.441519286Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:07.461175989Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:07.46121675Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:07.480331588Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:07.48037596Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:07.498063371Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:07.498100962Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:07.518126894Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:07.518180346Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:07.538098226Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:07.538139396Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:07.556720951Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:07.556754252Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:07.577021511Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:07.577063092Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:07.596496329Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:07.5965354Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:07.615093508Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:07.615129035Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:07.635898277Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:07.635936868Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:07.654210245Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:07.654252836Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:07.67588334Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:07.675920457Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:07.696111068Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:07.696162489Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:07.714948389Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:07.71498631Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:07.735262599Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/301424fa-c483-4ce9-92fb-3c8ee10e6f9f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:07.7353173Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/20dccfdd-e68a-468d-8629-7f7e0834aa1c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:07.735518306Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:07.735648689Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:07.836164751Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:07.836230513Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:07.836239193Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:07.836244864Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:07.836251404Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:07.836259074Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:07.836264834Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:07.836276134Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:07.836296525Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:07.836304085Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:07.836310165Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:07.836316105Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:07.836323386Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:07.836329946Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:07.836340386Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:07.836365647Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:07.836376377Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:07.836393197Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:07.836403608Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:07.836533301Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"0582975f-8782-4d5a-ac39-6f52bbef6505","remote_addr":"[fd00:1122:3344:101::1]:37752","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\", TimeNotSynchronized), (\"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203\", TimeNotSynchronized), (\"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd\", TimeNotSynchronized), (\"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1\", TimeNotSynchronized), (\"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660\", TimeNotSynchronized), (\"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7\", TimeNotSynchronized), (\"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec\", TimeNotSynchronized), (\"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1\", TimeNotSynchronized), (\"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb\", TimeNotSynchronized), (\"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1\", TimeNotSynchronized), (\"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b\", TimeNotSynchronized), (\"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603\", TimeNotSynchronized), (\"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f\", TimeNotSynchronized), (\"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\", TimeNotSynchronized), (\"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9\", TimeNotSynchronized), (\"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957\", TimeNotSynchronized), (\"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b\", TimeNotSynchronized), (\"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17\", TimeNotSynchronized), (\"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284\", TimeNotSynchronized)]","latency_us":499406,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:07.836750037Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"3.34857497s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0582975f-8782-4d5a-ac39-6f52bbef6505\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 17:46:07 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0582975f-8782-4d5a-ac39-6f52bbef6505\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 8ae14f97-bd01-42ad-87d5-23eae9194521, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: bc60810c-eece-4cbd-a0f8-d04271924b7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bf3fc707-b452-4d30-a92b-e56b4d4122ee, kind: Service { id: bc60810c-eece-4cbd-a0f8-d04271924b7c }, name: Name(\"ntp-bc60810c-eece-4cbd-a0f8-d04271924b7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 196])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: c168b27b-f467-4350-a348-4006ce85a213, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 15be01e5-1765-45f0-b521-da99b395bdb3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: d8a1310d-09c1-4844-91b7-509f93b30cfd, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 59731bd7-ffc9-493f-a33c-a1cf5ba2df30, kind: Service { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1 }, name: Name(\"external-dns-1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 32])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 09fdc6fd-8297-436e-903d-b18710599203, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 005b2740-84d5-473e-ae59-aa79fd2fe5b6, kind: Service { id: 09fdc6fd-8297-436e-903d-b18710599203 }, name: Name(\"external-dns-09fdc6fd-8297-436e-903d-b18710599203\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: c6edf68c-f558-4f51-9aed-6eb91fb3ad51, kind: Service { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52 }, name: Name(\"nexus-87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 117])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6088410e-a8fc-4d33-b593-6385e863291f, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e6898038-bf11-41c8-b971-6ff3eed6d38b, kind: Service { id: 6088410e-a8fc-4d33-b593-6385e863291f }, name: Name(\"nexus-6088410e-a8fc-4d33-b593-6385e863291f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 27])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 742e0330-77de-4723-a145-74751d41c73b, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: ffe6d41d-1349-42b2-a887-49d734eef422, kind: Service { id: 742e0330-77de-4723-a145-74751d41c73b }, name: Name(\"nexus-742e0330-77de-4723-a145-74751d41c73b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8d644923-5967-4351-a1ad-fe8a938f33e1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 740bd3e0-b30c-4033-968a-5451c8d5adcd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3286a4ca-fffd-4ed0-9c21-d528e1d23603, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b1925c8-3ae5-4372-a3de-213551574fe1, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 663bd1f7-7a29-484c-bfce-d312646b1660, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: c6d325c3-cddf-4528-8ba6-a083f6511284, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External } } } }, OmicronZoneConfig { id: c30b7bc1-fd10-4942-8055-ada48d2e3957, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External } } } }, OmicronZoneConfig { id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:11.177601261Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:11.177981721Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:11.178008941Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:11.202008168Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:11.20208509Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:11.223818766Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:11.223874918Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:11.245839841Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:11.245891472Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:11.265834922Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:11.265889934Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:11.28527544Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:11.285321581Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:11.305262801Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:11.305310492Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:11.323992477Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:11.324046871Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:11.343715924Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:11.343769466Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:11.363594433Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:11.363639314Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:11.383725328Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:11.38379301Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:11.404461089Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:11.40450848Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:11.427048428Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:11.427096149Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:11.447463411Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:11.447515972Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:11.467442442Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:11.467493603Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:11.487741492Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:11.487808353Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:11.507177649Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:11.50723742Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:11.526874103Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:11.526926354Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:11.547827229Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:11.54787372Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:11.568205721Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:11.568255032Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:11.58847509Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/301424fa-c483-4ce9-92fb-3c8ee10e6f9f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:11.588536921Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/20dccfdd-e68a-468d-8629-7f7e0834aa1c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:11.588765977Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:11.588909201Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:11.68854161Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:11.688617982Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:11.688630072Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:11.688637312Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:11.688643233Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:11.688651013Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:11.688656713Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:11.688671733Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:11.688677794Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:11.688692554Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:11.688698634Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:11.688702944Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:11.688715825Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:11.688721335Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:11.688726875Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:11.688737305Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:11.688743445Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:11.688750926Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:11.688755266Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:11.688802547Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"6f19aae3-4b78-4362-afe9-a24c38b0690b","remote_addr":"[fd00:1122:3344:101::1]:37752","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603\", TimeNotSynchronized), (\"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284\", TimeNotSynchronized), (\"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec\", TimeNotSynchronized), (\"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb\", TimeNotSynchronized), (\"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957\", TimeNotSynchronized), (\"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1\", TimeNotSynchronized), (\"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1\", TimeNotSynchronized), (\"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b\", TimeNotSynchronized), (\"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9\", TimeNotSynchronized), (\"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17\", TimeNotSynchronized), (\"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b\", TimeNotSynchronized), (\"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\", TimeNotSynchronized), (\"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f\", TimeNotSynchronized), (\"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1\", TimeNotSynchronized), (\"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\", TimeNotSynchronized), (\"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203\", TimeNotSynchronized), (\"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd\", TimeNotSynchronized), (\"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660\", TimeNotSynchronized), (\"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7\", TimeNotSynchronized)]","latency_us":512423,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:11.689012416Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"3.056985196s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6f19aae3-4b78-4362-afe9-a24c38b0690b\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 17:46:11 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6f19aae3-4b78-4362-afe9-a24c38b0690b\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:14.012785253Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:35537"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:14.012868275Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"98bb37d2-6ac7-4652-9b97-58d7d816dc2f","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:35537","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":48,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 8ae14f97-bd01-42ad-87d5-23eae9194521, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: bc60810c-eece-4cbd-a0f8-d04271924b7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bf3fc707-b452-4d30-a92b-e56b4d4122ee, kind: Service { id: bc60810c-eece-4cbd-a0f8-d04271924b7c }, name: Name(\"ntp-bc60810c-eece-4cbd-a0f8-d04271924b7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 196])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: c168b27b-f467-4350-a348-4006ce85a213, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 15be01e5-1765-45f0-b521-da99b395bdb3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: d8a1310d-09c1-4844-91b7-509f93b30cfd, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 59731bd7-ffc9-493f-a33c-a1cf5ba2df30, kind: Service { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1 }, name: Name(\"external-dns-1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 32])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 09fdc6fd-8297-436e-903d-b18710599203, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 005b2740-84d5-473e-ae59-aa79fd2fe5b6, kind: Service { id: 09fdc6fd-8297-436e-903d-b18710599203 }, name: Name(\"external-dns-09fdc6fd-8297-436e-903d-b18710599203\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: c6edf68c-f558-4f51-9aed-6eb91fb3ad51, kind: Service { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52 }, name: Name(\"nexus-87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 117])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6088410e-a8fc-4d33-b593-6385e863291f, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e6898038-bf11-41c8-b971-6ff3eed6d38b, kind: Service { id: 6088410e-a8fc-4d33-b593-6385e863291f }, name: Name(\"nexus-6088410e-a8fc-4d33-b593-6385e863291f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 27])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 742e0330-77de-4723-a145-74751d41c73b, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: ffe6d41d-1349-42b2-a887-49d734eef422, kind: Service { id: 742e0330-77de-4723-a145-74751d41c73b }, name: Name(\"nexus-742e0330-77de-4723-a145-74751d41c73b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8d644923-5967-4351-a1ad-fe8a938f33e1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 740bd3e0-b30c-4033-968a-5451c8d5adcd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3286a4ca-fffd-4ed0-9c21-d528e1d23603, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b1925c8-3ae5-4372-a3de-213551574fe1, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 663bd1f7-7a29-484c-bfce-d312646b1660, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: c6d325c3-cddf-4528-8ba6-a083f6511284, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External } } } }, OmicronZoneConfig { id: c30b7bc1-fd10-4942-8055-ada48d2e3957, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External } } } }, OmicronZoneConfig { id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:14.738475485Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:14.738825234Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:14.738837214Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:14.76204832Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:14.764510994Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:14.782588113Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:14.782646227Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:14.803783899Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:14.80383054Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:14.82378016Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:14.823826041Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:14.843929836Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:14.843993327Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:14.865424757Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:14.865484648Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:14.886169718Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:14.886223169Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:14.904883226Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:14.904929587Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:14.926290554Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:14.926338696Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:14.94682073Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:14.946868201Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:14.968174327Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:14.968229488Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:14.990423167Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:14.990468179Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:15.012481433Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:15.012529724Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:15.033535012Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:15.033590753Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:15.055770442Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:15.055819833Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:15.077498159Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:15.07754512Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:15.098290351Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:15.098338962Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:15.120168492Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:15.120218273Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:15.141505859Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:15.14155998Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:15.162451395Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/301424fa-c483-4ce9-92fb-3c8ee10e6f9f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:15.162527427Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/20dccfdd-e68a-468d-8629-7f7e0834aa1c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:15.162717852Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:15.162863466Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:15.259884397Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:15.259972999Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:15.259982019Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:15.25998792Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:15.25999601Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:15.26001147Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:15.26002424Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:15.260031891Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:15.260038281Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:15.260044401Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:15.260052261Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:15.260064962Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:15.260082792Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:15.260090722Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:15.260096593Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:15.260103993Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:15.260110163Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:15.260117583Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:15.260130993Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:15.260251717Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"452e37c0-a3b1-4c89-8620-5b4352c253d5","remote_addr":"[fd00:1122:3344:101::1]:37752","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1\", TimeNotSynchronized), (\"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f\", TimeNotSynchronized), (\"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603\", TimeNotSynchronized), (\"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1\", TimeNotSynchronized), (\"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd\", TimeNotSynchronized), (\"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec\", TimeNotSynchronized), (\"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9\", TimeNotSynchronized), (\"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7\", TimeNotSynchronized), (\"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb\", TimeNotSynchronized), (\"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b\", TimeNotSynchronized), (\"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17\", TimeNotSynchronized), (\"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\", TimeNotSynchronized), (\"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203\", TimeNotSynchronized), (\"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b\", TimeNotSynchronized), (\"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660\", TimeNotSynchronized), (\"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\", TimeNotSynchronized), (\"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284\", TimeNotSynchronized), (\"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957\", TimeNotSynchronized), (\"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1\", TimeNotSynchronized)]","latency_us":523049,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:15.260466522Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"5.489105675s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"452e37c0-a3b1-4c89-8620-5b4352c253d5\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 17:46:14 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"452e37c0-a3b1-4c89-8620-5b4352c253d5\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:16.23017146Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"4.094255488s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 8ae14f97-bd01-42ad-87d5-23eae9194521, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: bc60810c-eece-4cbd-a0f8-d04271924b7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bf3fc707-b452-4d30-a92b-e56b4d4122ee, kind: Service { id: bc60810c-eece-4cbd-a0f8-d04271924b7c }, name: Name(\"ntp-bc60810c-eece-4cbd-a0f8-d04271924b7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 196])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: c168b27b-f467-4350-a348-4006ce85a213, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 15be01e5-1765-45f0-b521-da99b395bdb3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: d8a1310d-09c1-4844-91b7-509f93b30cfd, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 59731bd7-ffc9-493f-a33c-a1cf5ba2df30, kind: Service { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1 }, name: Name(\"external-dns-1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 32])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 09fdc6fd-8297-436e-903d-b18710599203, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 005b2740-84d5-473e-ae59-aa79fd2fe5b6, kind: Service { id: 09fdc6fd-8297-436e-903d-b18710599203 }, name: Name(\"external-dns-09fdc6fd-8297-436e-903d-b18710599203\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: c6edf68c-f558-4f51-9aed-6eb91fb3ad51, kind: Service { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52 }, name: Name(\"nexus-87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 117])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6088410e-a8fc-4d33-b593-6385e863291f, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e6898038-bf11-41c8-b971-6ff3eed6d38b, kind: Service { id: 6088410e-a8fc-4d33-b593-6385e863291f }, name: Name(\"nexus-6088410e-a8fc-4d33-b593-6385e863291f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 27])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 742e0330-77de-4723-a145-74751d41c73b, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: ffe6d41d-1349-42b2-a887-49d734eef422, kind: Service { id: 742e0330-77de-4723-a145-74751d41c73b }, name: Name(\"nexus-742e0330-77de-4723-a145-74751d41c73b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8d644923-5967-4351-a1ad-fe8a938f33e1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 740bd3e0-b30c-4033-968a-5451c8d5adcd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3286a4ca-fffd-4ed0-9c21-d528e1d23603, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b1925c8-3ae5-4372-a3de-213551574fe1, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 663bd1f7-7a29-484c-bfce-d312646b1660, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: c6d325c3-cddf-4528-8ba6-a083f6511284, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External } } } }, OmicronZoneConfig { id: c30b7bc1-fd10-4942-8055-ada48d2e3957, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External } } } }, OmicronZoneConfig { id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:20.73578102Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:20.736081157Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:20.736100688Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:20.758192984Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:20.758262206Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:20.778241277Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:20.778305261Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:20.79788759Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:20.797936721Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:20.816168987Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:20.816229988Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:20.83433038Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:20.834399172Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:20.853433359Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:20.85348162Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:20.871245443Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:20.871293775Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:20.88911964Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:20.889169321Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:20.908430754Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:20.908476525Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:20.927379288Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:20.927431869Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:20.946660151Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:20.946728423Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:20.964762763Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:20.964837055Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:20.98379978Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:20.983865672Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:21.002035255Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:21.002075876Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:21.021096413Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:21.021143924Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:21.040299064Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:21.040342285Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:21.059769212Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:21.059830223Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:21.078960012Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:21.079010023Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:21.097608069Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:21.09767043Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:21.116933263Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/301424fa-c483-4ce9-92fb-3c8ee10e6f9f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:21.116994314Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/20dccfdd-e68a-468d-8629-7f7e0834aa1c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:21.11721081Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:21.117364964Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:21.210681558Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:21.2107417Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:21.21075596Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:21.210765141Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:21.210774341Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:21.210782941Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:21.210791992Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:21.210800332Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:21.210806882Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:21.210815162Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:21.210823382Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:21.210829602Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:21.210843433Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:21.210852303Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:21.210872743Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:21.210878444Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:21.210885114Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:21.210891904Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:21.210898724Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:21.210976776Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"eb3befad-d4be-4005-ae7a-b699610e2329","remote_addr":"[fd00:1122:3344:101::1]:37752","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd\", TimeNotSynchronized), (\"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1\", TimeNotSynchronized), (\"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f\", TimeNotSynchronized), (\"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284\", TimeNotSynchronized), (\"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7\", TimeNotSynchronized), (\"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603\", TimeNotSynchronized), (\"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb\", TimeNotSynchronized), (\"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17\", TimeNotSynchronized), (\"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\", TimeNotSynchronized), (\"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203\", TimeNotSynchronized), (\"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1\", TimeNotSynchronized), (\"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b\", TimeNotSynchronized), (\"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660\", TimeNotSynchronized), (\"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9\", TimeNotSynchronized), (\"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec\", TimeNotSynchronized), (\"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957\", TimeNotSynchronized), (\"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\", TimeNotSynchronized), (\"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b\", TimeNotSynchronized), (\"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1\", TimeNotSynchronized)]","latency_us":476360,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:21.211221593Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"3.173396656s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"eb3befad-d4be-4005-ae7a-b699610e2329\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 17:46:20 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"eb3befad-d4be-4005-ae7a-b699610e2329\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 8ae14f97-bd01-42ad-87d5-23eae9194521, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: bc60810c-eece-4cbd-a0f8-d04271924b7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bf3fc707-b452-4d30-a92b-e56b4d4122ee, kind: Service { id: bc60810c-eece-4cbd-a0f8-d04271924b7c }, name: Name(\"ntp-bc60810c-eece-4cbd-a0f8-d04271924b7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 196])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: c168b27b-f467-4350-a348-4006ce85a213, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 15be01e5-1765-45f0-b521-da99b395bdb3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: d8a1310d-09c1-4844-91b7-509f93b30cfd, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 59731bd7-ffc9-493f-a33c-a1cf5ba2df30, kind: Service { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1 }, name: Name(\"external-dns-1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 32])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 09fdc6fd-8297-436e-903d-b18710599203, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 005b2740-84d5-473e-ae59-aa79fd2fe5b6, kind: Service { id: 09fdc6fd-8297-436e-903d-b18710599203 }, name: Name(\"external-dns-09fdc6fd-8297-436e-903d-b18710599203\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: c6edf68c-f558-4f51-9aed-6eb91fb3ad51, kind: Service { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52 }, name: Name(\"nexus-87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 117])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6088410e-a8fc-4d33-b593-6385e863291f, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e6898038-bf11-41c8-b971-6ff3eed6d38b, kind: Service { id: 6088410e-a8fc-4d33-b593-6385e863291f }, name: Name(\"nexus-6088410e-a8fc-4d33-b593-6385e863291f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 27])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 742e0330-77de-4723-a145-74751d41c73b, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: ffe6d41d-1349-42b2-a887-49d734eef422, kind: Service { id: 742e0330-77de-4723-a145-74751d41c73b }, name: Name(\"nexus-742e0330-77de-4723-a145-74751d41c73b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8d644923-5967-4351-a1ad-fe8a938f33e1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 740bd3e0-b30c-4033-968a-5451c8d5adcd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3286a4ca-fffd-4ed0-9c21-d528e1d23603, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b1925c8-3ae5-4372-a3de-213551574fe1, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 663bd1f7-7a29-484c-bfce-d312646b1660, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: c6d325c3-cddf-4528-8ba6-a083f6511284, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External } } } }, OmicronZoneConfig { id: c30b7bc1-fd10-4942-8055-ada48d2e3957, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External } } } }, OmicronZoneConfig { id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:24.377809831Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:24.378209171Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:24.378232802Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:24.40192071Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:24.401981942Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:24.4241655Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:24.424222982Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:24.444746307Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:24.444816199Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:24.464740679Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:24.46479628Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:24.48512866Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:24.485185552Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:24.505007449Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:24.505062291Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:24.526583172Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:24.526637003Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:24.547874628Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:24.547937149Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:24.568695081Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:24.568755792Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:24.590838168Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:24.5908868Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:24.61122747Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:24.611296842Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:24.631496139Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:24.631567641Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:24.65301742Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:24.653069782Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:24.673377522Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:24.673445663Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:24.693725462Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:24.693785124Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:24.714540465Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:24.714589257Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:24.735079911Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:24.735142633Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:24.754222601Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:24.754278462Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:24.776866371Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:24.776925013Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:24.79980159Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/301424fa-c483-4ce9-92fb-3c8ee10e6f9f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:24.799858791Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/20dccfdd-e68a-468d-8629-7f7e0834aa1c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:24.800069667Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:24.80020681Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:24.899442769Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:24.899533602Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:24.899567683Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:24.899582663Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:24.899591513Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:24.899598363Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:24.899606833Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:24.899613834Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:24.899622654Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:24.899631104Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:24.899645974Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:24.899654405Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:24.899662875Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:24.899667695Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:24.899674395Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:24.899684346Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:24.899693036Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:24.899698366Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:24.899711076Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:24.899777938Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"2c7056af-12ff-46fb-8d66-9fb9996b8e8e","remote_addr":"[fd00:1122:3344:101::1]:37752","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f\", TimeNotSynchronized), (\"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603\", TimeNotSynchronized), (\"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203\", TimeNotSynchronized), (\"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b\", TimeNotSynchronized), (\"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd\", TimeNotSynchronized), (\"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284\", TimeNotSynchronized), (\"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7\", TimeNotSynchronized), (\"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb\", TimeNotSynchronized), (\"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957\", TimeNotSynchronized), (\"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17\", TimeNotSynchronized), (\"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\", TimeNotSynchronized), (\"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec\", TimeNotSynchronized), (\"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1\", TimeNotSynchronized), (\"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1\", TimeNotSynchronized), (\"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b\", TimeNotSynchronized), (\"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660\", TimeNotSynchronized), (\"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9\", TimeNotSynchronized), (\"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\", TimeNotSynchronized), (\"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1\", TimeNotSynchronized)]","latency_us":523241,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:24.900025484Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"7.799738611s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2c7056af-12ff-46fb-8d66-9fb9996b8e8e\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 17:46:24 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2c7056af-12ff-46fb-8d66-9fb9996b8e8e\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 8ae14f97-bd01-42ad-87d5-23eae9194521, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: bc60810c-eece-4cbd-a0f8-d04271924b7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bf3fc707-b452-4d30-a92b-e56b4d4122ee, kind: Service { id: bc60810c-eece-4cbd-a0f8-d04271924b7c }, name: Name(\"ntp-bc60810c-eece-4cbd-a0f8-d04271924b7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 196])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: c168b27b-f467-4350-a348-4006ce85a213, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 15be01e5-1765-45f0-b521-da99b395bdb3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: d8a1310d-09c1-4844-91b7-509f93b30cfd, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 59731bd7-ffc9-493f-a33c-a1cf5ba2df30, kind: Service { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1 }, name: Name(\"external-dns-1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 32])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 09fdc6fd-8297-436e-903d-b18710599203, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 005b2740-84d5-473e-ae59-aa79fd2fe5b6, kind: Service { id: 09fdc6fd-8297-436e-903d-b18710599203 }, name: Name(\"external-dns-09fdc6fd-8297-436e-903d-b18710599203\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: c6edf68c-f558-4f51-9aed-6eb91fb3ad51, kind: Service { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52 }, name: Name(\"nexus-87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 117])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6088410e-a8fc-4d33-b593-6385e863291f, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e6898038-bf11-41c8-b971-6ff3eed6d38b, kind: Service { id: 6088410e-a8fc-4d33-b593-6385e863291f }, name: Name(\"nexus-6088410e-a8fc-4d33-b593-6385e863291f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 27])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 742e0330-77de-4723-a145-74751d41c73b, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: ffe6d41d-1349-42b2-a887-49d734eef422, kind: Service { id: 742e0330-77de-4723-a145-74751d41c73b }, name: Name(\"nexus-742e0330-77de-4723-a145-74751d41c73b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8d644923-5967-4351-a1ad-fe8a938f33e1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 740bd3e0-b30c-4033-968a-5451c8d5adcd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3286a4ca-fffd-4ed0-9c21-d528e1d23603, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b1925c8-3ae5-4372-a3de-213551574fe1, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 663bd1f7-7a29-484c-bfce-d312646b1660, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: c6d325c3-cddf-4528-8ba6-a083f6511284, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External } } } }, OmicronZoneConfig { id: c30b7bc1-fd10-4942-8055-ada48d2e3957, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External } } } }, OmicronZoneConfig { id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:32.680198849Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:32.680582689Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:32.68059712Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:32.703122097Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:32.703178349Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:32.723620162Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:32.723667803Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:32.74155294Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:32.741621602Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:32.759640692Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:32.759696183Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:32.7791165Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:32.779183222Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:32.797303214Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:32.797354206Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:32.817017079Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:32.81707301Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:32.834989758Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:32.835040399Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:32.854747633Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:32.854798564Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:32.87610774Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:32.876153071Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:32.895111336Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:32.895158067Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:32.914125972Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:32.914175843Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:32.93285562Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:32.932897842Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:32.951133697Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:32.951178478Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:32.970053011Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:32.970092462Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:32.988666027Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:32.988711458Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:33.007460887Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:33.007506518Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:33.026855733Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:33.026915474Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:33.045574351Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:33.045633503Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:33.063828887Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/301424fa-c483-4ce9-92fb-3c8ee10e6f9f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:33.063883789Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/20dccfdd-e68a-468d-8629-7f7e0834aa1c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:33.064113495Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:33.064271259Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:33.155290453Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:33.155356185Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:33.155370385Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:33.155377826Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:33.155385286Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:33.155392356Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:33.155418077Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:33.155423497Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:33.155430827Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:33.155436777Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:33.155444407Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:33.155451847Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:33.155458698Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:33.155467448Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:33.155484588Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:33.155491979Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:33.155506879Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:33.155515669Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:33.1555239Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:33.155638902Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"4b65e5cd-722e-4afe-b0dc-30db340459c3","remote_addr":"[fd00:1122:3344:101::1]:37752","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b\", TimeNotSynchronized), (\"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\", TimeNotSynchronized), (\"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660\", TimeNotSynchronized), (\"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284\", TimeNotSynchronized), (\"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb\", TimeNotSynchronized), (\"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957\", TimeNotSynchronized), (\"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17\", TimeNotSynchronized), (\"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603\", TimeNotSynchronized), (\"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7\", TimeNotSynchronized), (\"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b\", TimeNotSynchronized), (\"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1\", TimeNotSynchronized), (\"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd\", TimeNotSynchronized), (\"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9\", TimeNotSynchronized), (\"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203\", TimeNotSynchronized), (\"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\", TimeNotSynchronized), (\"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec\", TimeNotSynchronized), (\"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1\", TimeNotSynchronized), (\"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f\", TimeNotSynchronized), (\"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1\", TimeNotSynchronized)]","latency_us":476587,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:33.155886509Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"3.685750344s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4b65e5cd-722e-4afe-b0dc-30db340459c3\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 17:46:32 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4b65e5cd-722e-4afe-b0dc-30db340459c3\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:35.308780373Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"8.142177937s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 8ae14f97-bd01-42ad-87d5-23eae9194521, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: bc60810c-eece-4cbd-a0f8-d04271924b7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bf3fc707-b452-4d30-a92b-e56b4d4122ee, kind: Service { id: bc60810c-eece-4cbd-a0f8-d04271924b7c }, name: Name(\"ntp-bc60810c-eece-4cbd-a0f8-d04271924b7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 196])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: c168b27b-f467-4350-a348-4006ce85a213, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 15be01e5-1765-45f0-b521-da99b395bdb3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: d8a1310d-09c1-4844-91b7-509f93b30cfd, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 59731bd7-ffc9-493f-a33c-a1cf5ba2df30, kind: Service { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1 }, name: Name(\"external-dns-1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 32])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 09fdc6fd-8297-436e-903d-b18710599203, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 005b2740-84d5-473e-ae59-aa79fd2fe5b6, kind: Service { id: 09fdc6fd-8297-436e-903d-b18710599203 }, name: Name(\"external-dns-09fdc6fd-8297-436e-903d-b18710599203\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: c6edf68c-f558-4f51-9aed-6eb91fb3ad51, kind: Service { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52 }, name: Name(\"nexus-87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 117])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6088410e-a8fc-4d33-b593-6385e863291f, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e6898038-bf11-41c8-b971-6ff3eed6d38b, kind: Service { id: 6088410e-a8fc-4d33-b593-6385e863291f }, name: Name(\"nexus-6088410e-a8fc-4d33-b593-6385e863291f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 27])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 742e0330-77de-4723-a145-74751d41c73b, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: ffe6d41d-1349-42b2-a887-49d734eef422, kind: Service { id: 742e0330-77de-4723-a145-74751d41c73b }, name: Name(\"nexus-742e0330-77de-4723-a145-74751d41c73b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8d644923-5967-4351-a1ad-fe8a938f33e1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 740bd3e0-b30c-4033-968a-5451c8d5adcd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3286a4ca-fffd-4ed0-9c21-d528e1d23603, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b1925c8-3ae5-4372-a3de-213551574fe1, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 663bd1f7-7a29-484c-bfce-d312646b1660, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: c6d325c3-cddf-4528-8ba6-a083f6511284, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External } } } }, OmicronZoneConfig { id: c30b7bc1-fd10-4942-8055-ada48d2e3957, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External } } } }, OmicronZoneConfig { id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:36.832777971Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:36.833240293Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:36.833255593Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:36.856614317Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:36.856668314Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:36.877474367Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:36.877534893Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:36.898043993Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:36.898107375Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:36.917667925Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:36.917712256Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:36.935519651Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:36.935568342Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:36.953605512Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:36.953664574Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:36.972191447Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:36.972242629Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:36.990571087Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:36.990636949Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:37.008746271Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:37.008805972Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:37.027127817Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:37.027176042Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:37.045426198Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:37.045474439Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:37.065783859Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:37.06583723Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:37.084260781Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:37.084331643Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:37.104231632Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:37.104294234Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:37.123161936Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:37.123227838Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:37.141918095Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:37.141965246Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:37.160785095Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:37.160838319Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:37.178750981Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:37.178810268Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:37.197101065Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:37.197145536Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:37.216486381Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/301424fa-c483-4ce9-92fb-3c8ee10e6f9f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:37.216549842Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/20dccfdd-e68a-468d-8629-7f7e0834aa1c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:37.216795639Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:37.216940802Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:37.309800445Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:37.309863246Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:37.309881367Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:37.309901267Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:37.309908388Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:37.309920648Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:37.309927598Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:37.309934818Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:37.309958169Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:37.309967119Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:37.309981069Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:37.30998995Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:37.30999544Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:37.3100102Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:37.310019421Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:37.310026691Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:37.310035661Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:37.310042741Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:37.310049781Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2984","zone":"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:37.310153574Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"b94f954e-136f-45de-adad-3b3570e998a7","remote_addr":"[fd00:1122:3344:101::1]:37752","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660\", TimeNotSynchronized), (\"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b\", TimeNotSynchronized), (\"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203\", TimeNotSynchronized), (\"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9\", TimeNotSynchronized), (\"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603\", TimeNotSynchronized), (\"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284\", TimeNotSynchronized), (\"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd\", TimeNotSynchronized), (\"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b\", TimeNotSynchronized), (\"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17\", TimeNotSynchronized), (\"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\", TimeNotSynchronized), (\"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\", TimeNotSynchronized), (\"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f\", TimeNotSynchronized), (\"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec\", TimeNotSynchronized), (\"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7\", TimeNotSynchronized), (\"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1\", TimeNotSynchronized), (\"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb\", TimeNotSynchronized), (\"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957\", TimeNotSynchronized), (\"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1\", TimeNotSynchronized), (\"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1\", TimeNotSynchronized)]","latency_us":478442,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:37.310424571Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"11.210331039s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b94f954e-136f-45de-adad-3b3570e998a7\", \"content-length\": \"124\", \"date\": \"Fri, 19 Jul 2024 17:46:36 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b94f954e-136f-45de-adad-3b3570e998a7\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:43.96533157Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:57738"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:43.965376572Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"e353c946-5a80-4750-bf3b-451bfa707ef8","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:57738","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":30,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 8ae14f97-bd01-42ad-87d5-23eae9194521, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: bc60810c-eece-4cbd-a0f8-d04271924b7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: bf3fc707-b452-4d30-a92b-e56b4d4122ee, kind: Service { id: bc60810c-eece-4cbd-a0f8-d04271924b7c }, name: Name(\"ntp-bc60810c-eece-4cbd-a0f8-d04271924b7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 128, 196])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: c168b27b-f467-4350-a348-4006ce85a213, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 15be01e5-1765-45f0-b521-da99b395bdb3, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: d8a1310d-09c1-4844-91b7-509f93b30cfd, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: 59731bd7-ffc9-493f-a33c-a1cf5ba2df30, kind: Service { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1 }, name: Name(\"external-dns-1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 32])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 09fdc6fd-8297-436e-903d-b18710599203, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 005b2740-84d5-473e-ae59-aa79fd2fe5b6, kind: Service { id: 09fdc6fd-8297-436e-903d-b18710599203 }, name: Name(\"external-dns-09fdc6fd-8297-436e-903d-b18710599203\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: c6edf68c-f558-4f51-9aed-6eb91fb3ad51, kind: Service { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52 }, name: Name(\"nexus-87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 117])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6088410e-a8fc-4d33-b593-6385e863291f, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e6898038-bf11-41c8-b971-6ff3eed6d38b, kind: Service { id: 6088410e-a8fc-4d33-b593-6385e863291f }, name: Name(\"nexus-6088410e-a8fc-4d33-b593-6385e863291f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 27])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 742e0330-77de-4723-a145-74751d41c73b, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: ffe6d41d-1349-42b2-a887-49d734eef422, kind: Service { id: 742e0330-77de-4723-a145-74751d41c73b }, name: Name(\"nexus-742e0330-77de-4723-a145-74751d41c73b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 8d644923-5967-4351-a1ad-fe8a938f33e1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 740bd3e0-b30c-4033-968a-5451c8d5adcd, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3286a4ca-fffd-4ed0-9c21-d528e1d23603, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 0b1925c8-3ae5-4372-a3de-213551574fe1, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External } } } }, OmicronZoneConfig { id: 663bd1f7-7a29-484c-bfce-d312646b1660, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External } } } }, OmicronZoneConfig { id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External } } } }, OmicronZoneConfig { id: c6d325c3-cddf-4528-8ba6-a083f6511284, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External } } } }, OmicronZoneConfig { id: c30b7bc1-fd10-4942-8055-ada48d2e3957, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External } } } }, OmicronZoneConfig { id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:48.491970623Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:48.492295487Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:48.492307467Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8ae14f97-bd01-42ad-87d5-23eae9194521, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:48.515059451Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:48.515132803Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 98b939a3-2514-4766-b854-3b3ddf8b23f9, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:48.535889904Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:48.535956246Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 8f9c564d-5823-4b85-a0ff-2de7f54b602e, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:48.555987719Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:48.55604733Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 27c74baf-8069-4296-ac26-2f4818d2ab3a, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:48.574388359Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:48.57444613Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c168b27b-f467-4350-a348-4006ce85a213, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:48.592783379Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:48.59283087Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 15be01e5-1765-45f0-b521-da99b395bdb3, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:48.609993997Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:48.610042999Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ecfb00b6-cf92-4e9f-ab59-5520a05aa0d0, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:48.628017998Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:48.628071659Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d8a1310d-09c1-4844-91b7-509f93b30cfd, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:48.645488063Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:48.645547177Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:48.664408587Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:48.664467529Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 09fdc6fd-8297-436e-903d-b18710599203, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:48.682509759Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:48.682566791Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2212adda-2fcf-4d59-888c-2223a4fe3f6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:48.700644552Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:48.700701004Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0b1925c8-3ae5-4372-a3de-213551574fe1, dataset_name: DatasetName { pool_name: ZpoolName { id: 988e775a-ddc4-455d-bc4e-f2e3c228cc74 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:48.718796446Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:48.718851527Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 663bd1f7-7a29-484c-bfce-d312646b1660, dataset_name: DatasetName { pool_name: ZpoolName { id: f3be927c-0009-45ce-8f85-0c94f9b67b8f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:48.737185346Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:48.737239977Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 605558ae-0f1e-47a7-b88c-f397548d4ea1, dataset_name: DatasetName { pool_name: ZpoolName { id: ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:48.756364376Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:48.756428558Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: bd7f3d92-4497-48dd-9f51-160ce4d7f4d9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4c50992d-c53d-42b7-a46f-5340e3aee556 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:48.776556073Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:48.776622355Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c6d325c3-cddf-4528-8ba6-a083f6511284, dataset_name: DatasetName { pool_name: ZpoolName { id: 7c02949e-12b1-4d82-9ac3-2efd807aef95 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:48.794605833Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:48.796831112Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 6fa8dbd2-a741-440b-98a2-b60bf983cdc7, dataset_name: DatasetName { pool_name: ZpoolName { id: 765bc25e-72d0-4d30-8cfe-a58b7db5c2e8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:48.81246385Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:48.812521761Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb, dataset_name: DatasetName { pool_name: ZpoolName { id: 696d009b-6d0c-4082-b81b-231c6c85d568 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:48.830142341Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:48.830192982Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: c30b7bc1-fd10-4942-8055-ada48d2e3957, dataset_name: DatasetName { pool_name: ZpoolName { id: 311acf6c-db4a-40cb-bd1f-e33d2bc8747f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:48.847804461Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:48.847858593Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17, dataset_name: DatasetName { pool_name: ZpoolName { id: 41faf794-cdf9-49a0-b529-4c18e335c28c (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:48.865571865Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/301424fa-c483-4ce9-92fb-3c8ee10e6f9f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:48.865628486Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/20dccfdd-e68a-468d-8629-7f7e0834aa1c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:48.865898953Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:48.866045137Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:48.993886992Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:263"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:48.994115718Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:48.994158029Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:48.99420596Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:48.995525625Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:263"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.041209157Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.043047985Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.043141587Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.043178328Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.043210889Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.191429286Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:263"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.193238113Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:263"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.243117834Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:263"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.243370611Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.243432702Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.243482814Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.243532805Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.243556385Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.243565986Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Configuring new Omicron zone: oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.243632478Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.248112995Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.252496229Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.257297104Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.262159171Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.2671041Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.272017318Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.276482395Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.39358956Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.397409189Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.401587378Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.406071025Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.410675485Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.4154426Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.420785159Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.425471611Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.432822283Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.438395456Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.443366958Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.449022296Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.455387251Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.460734101Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.467373415Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.47370199Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.480162308Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.485330473Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.49134692Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.498062175Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.563460731Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.646434386Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.772301369Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.832558011Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.888770598Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.890633556Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:305"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.890657947Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.940398835Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:426","port":"Port { inner: PortInner(PortData { name: \"opte1\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 199, 184]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.940430335Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1210","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.941935895Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:305"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.941962625Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.995077601Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:426","port":"Port { inner: PortInner(PortData { name: \"opte2\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 163, 117]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.995111872Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1210","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.996652602Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:305"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:49.996672423Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:50.067227753Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:426","port":"Port { inner: PortInner(PortData { name: \"opte3\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 171, 27]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:50.067258224Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1210","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:50.068763593Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:305"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:50.068783224Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:50.120654807Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:426","port":"Port { inner: PortInner(PortData { name: \"opte4\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 187, 6]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:50.120691208Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1210","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:50.122657369Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:305"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:50.12267817Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:50.176535855Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:426","port":"Port { inner: PortInner(PortData { name: \"opte5\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 175, 32]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:50.176567196Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1210","zone_type":"external_dns"} {"msg":"no nat entry found for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:50.177305495Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:50.177322745Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:50.179659346Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:50.179674367Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:50.179750329Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:50.179759949Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:50.179822751Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:50.179832661Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:50.180059357Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:50.180071497Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:50.18207982Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:50.183666001Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"creation of nat entry successful for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:50.183682761Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:50.231198671Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"creation of nat entry successful for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:50.231471178Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"creation of nat entry successful for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:50.236594592Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:50.239692312Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:50.239714973Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"creation of nat entry successful for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:50.239816696Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Configuring new Omicron zone: oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:50.28789533Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:50.328307884Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Configuring new Omicron zone: oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:50.441849826Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:50.448163371Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:50.453468349Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_nexus_742e0330-77de-4723-a145-74751d41c73b","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:50.460282777Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:50.467655599Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:50.473867782Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:50.48339296Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:50.494046458Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_742e0330-77de-4723-a145-74751d41c73b","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:50.500457115Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:53.206858729Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:53.207083415Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Profile for oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:53.301907335Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:53.302116824Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Profile for oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:53.357805237Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:53.358027223Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Profile for oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:53.36944942Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:53.369694187Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Profile for oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:53.377658215Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:53.377789238Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Profile for oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:53.418191882Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:53.418411848Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Profile for oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:53.461766379Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:53.462149969Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Profile for oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:53.470209569Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:53.470397584Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"wait for service Some(\"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603\") failed: Property not found. retry in 42.671196ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:53.509758321Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:53.513807366Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:53.514019952Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Profile for oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:53.524033713Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:53.524223398Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"wait for service Some(\"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957\") failed: Property not found. retry in 49.586827ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:53.544729383Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603\") failed: Property not found. retry in 133.276912ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:53.565126935Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284\") failed: Property not found. retry in 27.674166ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:53.583974637Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb\") failed: Property not found. retry in 65.300412ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:53.59596897Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9\") failed: Property not found. retry in 32.90706ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:53.606673779Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957\") failed: Property not found. retry in 128.500648ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:53.617118392Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284\") failed: Property not found. retry in 101.459504ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:53.6312231Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9\") failed: Property not found. retry in 116.832572ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:53.745423159Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284\") failed: Property not found. retry in 253.062913ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:53.887786752Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957\") failed: Property not found. retry in 265.605423ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:53.954098152Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603\") failed: Property not found. retry in 260.559496ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:56.461704569Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb\") failed: Property not found. retry in 60.492294ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:56.530261267Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17\") failed: Property not found. retry in 26.463018ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:56.659731165Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660\") failed: Property not found. retry in 34.817939ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:56.92481689Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9\") failed: Property not found. retry in 218.893169ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:57.023260678Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284\") failed: Property not found. retry in 333.13421ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:57.127066516Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957\") failed: Property not found. retry in 377.990133ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:57.152274974Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603\") failed: Property not found. retry in 288.696625ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:57.174876284Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7\") failed: Property not found. retry in 30.461889ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:57.201669883Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb\") failed: Property not found. retry in 228.830816ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:57.27609066Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17\") failed: Property not found. retry in 85.674858ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:57.448910982Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1\") failed: Property not found. retry in 63.310489ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:57.526462245Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1\") failed: Property not found. retry in 73.418329ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:57.600521448Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660\") failed: Property not found. retry in 56.554687ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:57.620057697Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9\") failed: Property not found. retry in 545.23398ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:57.644906295Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284\") failed: Property not found. retry in 751.606524ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:57.659476416Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957\") failed: Property not found. retry in 678.077683ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:57.674910598Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603\") failed: Property not found. retry in 572.378501ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:57.689621622Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7\") failed: Property not found. retry in 61.334974ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:57.707285223Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb\") failed: Property not found. retry in 439.185821ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:57.723178508Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17\") failed: Property not found. retry in 186.006606ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:57.738272341Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1\") failed: Property not found. retry in 91.210483ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:57.754587687Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:57.756162428Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:57.757588545Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1\") failed: Property not found. retry in 131.38106ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:57.774114246Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660\") failed: Property not found. retry in 181.100652ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:57.789003775Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7\") failed: Property not found. retry in 213.117473ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:57.803729409Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:57.803906234Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:57.812246501Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Profile for oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:57.829038169Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:57.829430739Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"wait for service Some(\"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1\") failed: Property not found. retry in 207.912431ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:57.862034869Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1\") failed: Property not found. retry in 284.904366ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:57.925304989Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17\") failed: Property not found. retry in 217.528863ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:57.942253182Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660\") failed: Property not found. retry in 437.962178ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:57.9924181Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7\") failed: Property not found. retry in 283.244365ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:58.033347908Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1\") failed: Property not found. retry in 228.191289ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:58.086295369Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17\") failed: Property not found. retry in 1.197053231s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:58.176815661Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb\") failed: Property not found. retry in 1.007683317s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:58.192794468Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9\") failed: Property not found. retry in 655.531247ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:58.206827304Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec\") failed: Property not found. retry in 58.172534ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:58.220076679Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1\") failed: Property not found. retry in 64.021528ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:58.232034951Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd\") failed: Property not found. retry in 72.925618ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:58.244143877Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1\") failed: Property not found. retry in 285.579094ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:58.2565237Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603\") failed: Property not found. retry in 798.411374ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:58.274844688Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec\") failed: Property not found. retry in 118.799427ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:58.289062449Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1\") failed: Property not found. retry in 101.775719ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:58.321334221Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7\") failed: Property not found. retry in 1.049237261s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:58.345667846Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1\") failed: Property not found. retry in 711.603215ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:58.374624181Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd\") failed: Property not found. retry in 73.150747ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:58.393446292Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957\") failed: Property not found. retry in 1.142180569s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:58.414577014Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284\") failed: Property not found. retry in 1.101909839s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:58.432429689Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec\") failed: Property not found. retry in 287.900089ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:58.448017586Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660\") failed: Property not found. retry in 1.092994235s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:58.467204117Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1\") failed: Property not found. retry in 225.024608ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:58.481259573Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd\") failed: Property not found. retry in 279.304853ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:58.495859724Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:58.507680902Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): operation timed out\"","delay":"19.258354202s"} {"msg":"wait for service Some(\"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1\") failed: Property not found. retry in 494.117672ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:58.564207587Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1\") failed: Property not found. retry in 406.788016ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:58.790126661Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec\") failed: Property not found. retry in 330.416092ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:58.812780842Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd\") failed: Property not found. retry in 599.106685ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:58.83377665Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9\") failed: Property not found. retry in 500.438886ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:58.880534829Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1\") failed: Property not found. retry in 516.893883ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:59.079278479Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603\") failed: Property not found. retry in 557.230679ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:59.102184804Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1\") failed: Property not found. retry in 1.228958481s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:59.119931496Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec\") failed: Property not found. retry in 794.317636ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:59.158749626Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1\") failed: Property not found. retry in 581.700211ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:59.212022642Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb\") failed: Property not found. retry in 567.980403ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:59.227422843Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17\") failed: Property not found. retry in 1.416209326s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:59.38989754Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9\") failed: Property not found. retry in 860.330191ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:59.404565362Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7\") failed: Property not found. retry in 1.090737903s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:59.420466785Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd\") failed: Property not found. retry in 556.403996ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:59.449042349Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284\") failed: Property not found. retry in 838.865868ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:59.547392847Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957\") failed: Property not found. retry in 813.349569ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:59.568476796Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660\") failed: Property not found. retry in 1.345370503s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:59.580201071Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:59.58129837Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:59.581776422Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:59.581842044Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:46:59.588161308Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"wait for service Some(\"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1\") failed: Property not found. retry in 1.377391407s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:59.608755354Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603\") failed: Property not found. retry in 1.447664736s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:59.672508123Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1\") failed: Property not found. retry in 1.295998986s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:59.813580103Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\") failed: Property not found. retry in 56.863795ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:59.828897171Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb\") failed: Property not found. retry in 543.157729ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:59.846065877Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203\") failed: Property not found. retry in 73.410066ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:59.860501913Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\") failed: Property not found. retry in 69.807334ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:59.902556797Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203\") failed: Property not found. retry in 121.782092ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:46:59.986732157Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\") failed: Property not found. retry in 168.837796ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:00.013039721Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec\") failed: Property not found. retry in 1.395804975s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:00.046526743Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd\") failed: Property not found. retry in 975.233418ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:00.070203819Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203\") failed: Property not found. retry in 112.119358ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:00.135728223Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\") failed: Property not found. retry in 572.42778ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:00.202262644Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203\") failed: Property not found. retry in 204.606199ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:00.372719249Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9\") failed: Property not found. retry in 758.353744ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:00.400451401Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1\") failed: Property not found. retry in 1.370846755s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:00.461588831Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284\") failed: Property not found. retry in 769.332724ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:00.547938938Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957\") failed: Property not found. retry in 746.931419ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:00.57067966Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7\") failed: Property not found. retry in 1.120380065s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:00.588174205Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb\") failed: Property not found. retry in 582.269227ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:00.605441984Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203\") failed: Property not found. retry in 937.646348ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:00.620389433Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\") failed: Property not found. retry in 744.638989ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:00.790781756Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17\") failed: Property not found. retry in 1.014483368s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:00.819165314Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660\") failed: Property not found. retry in 975.57683ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:00.941545998Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1\") failed: Property not found. retry in 644.881737ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:01.003317725Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd\") failed: Property not found. retry in 1.400078817s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:01.059540308Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1\") failed: Property not found. retry in 1.435191729s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:01.121743767Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603\") failed: Property not found. retry in 858.593535ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:01.133575765Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9\") failed: Property not found. retry in 1.49370871s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:01.169271553Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb\") failed: Property not found. retry in 1.377637436s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:01.199755816Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_742e0330-77de-4723-a145-74751d41c73b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:01.281612136Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:01.281971075Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:01.282011536Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:01.282033427Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:01.289245095Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:01.300633801Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"wait for service Some(\"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284\") failed: Property not found. retry in 615.071809ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:01.328842485Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957\") failed: Property not found. retry in 573.491992ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:01.339151413Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec\") failed: Property not found. retry in 1.441529806s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:01.456220079Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\") failed: Property not found. retry in 1.163177448s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:01.611680833Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b\") failed: Property not found. retry in 61.963012ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:01.639427215Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\") failed: Property not found. retry in 60.681008ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:01.663105681Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203\") failed: Property not found. retry in 1.013960173s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:01.684824686Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f\") failed: Property not found. retry in 30.005659ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:01.702879086Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1\") failed: Property not found. retry in 644.520072ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:01.721266235Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b\") failed: Property not found. retry in 57.379735ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:01.749711904Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7\") failed: Property not found. retry in 1.437060697s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:01.774640793Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\") failed: Property not found. retry in 79.781931ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:01.809604163Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f\") failed: Property not found. retry in 106.355821ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:01.829506411Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b\") failed: Property not found. retry in 143.899668ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:01.8579372Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17\") failed: Property not found. retry in 1.105750712s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:01.883545706Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1\") failed: Property not found. retry in 725.450162ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:01.904226555Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\") failed: Property not found. retry in 210.804334ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:01.929832791Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660\") failed: Property not found. retry in 837.766378ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:01.953578888Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284\") failed: Property not found. retry in 1.036437053s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:01.970657743Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957\") failed: Property not found. retry in 987.64218ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:01.986061193Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f\") failed: Property not found. retry in 206.639489ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:02.005725845Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b\") failed: Property not found. retry in 226.096906ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:02.04013328Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603\") failed: Property not found. retry in 810.729514ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:02.064521855Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:02.069241458Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:02.069565636Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"wait for service Some(\"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\") failed: Property not found. retry in 505.805452ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:02.189054265Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f\") failed: Property not found. retry in 282.444076ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:02.227098235Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b\") failed: Property not found. retry in 1.141627052s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:02.266818358Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b\") failed: Property not found. retry in 34.168512ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:02.359254563Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1\") failed: Property not found. retry in 771.385474ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:02.379847179Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b\") failed: Property not found. retry in 62.294285ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:02.403762851Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd\") failed: Property not found. retry in 1.147178293s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:02.472470568Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b\") failed: Property not found. retry in 260.332708ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:02.482606412Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f\") failed: Property not found. retry in 1.09342023s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:02.519705468Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1\") failed: Property not found. retry in 519.40274ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:02.567345487Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb\") failed: Property not found. retry in 665.710402ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:02.589315219Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1\") failed: Property not found. retry in 626.02901ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:02.661467456Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9\") failed: Property not found. retry in 1.330352943s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:02.697458462Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\") failed: Property not found. retry in 962.838838ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:02.710731707Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203\") failed: Property not found. retry in 587.008173ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:02.728383977Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b\") failed: Property not found. retry in 486.380233ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:02.76157187Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\") failed: Property not found. retry in 514.479278ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:02.791956141Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660\") failed: Property not found. retry in 625.34014ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:02.804044465Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603\") failed: Property not found. retry in 871.780212ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:02.890574806Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec\") failed: Property not found. retry in 737.352229ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:02.910010792Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957\") failed: Property not found. retry in 729.669749ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:02.988213306Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17\") failed: Property not found. retry in 670.234051ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:03.00333368Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284\") failed: Property not found. retry in 1.189073053s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:03.021363929Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1\") failed: Property not found. retry in 542.964362ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:03.10057286Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1\") failed: Property not found. retry in 1.381790167s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:03.164915594Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7\") failed: Property not found. retry in 623.900142ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:03.226367262Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b\") failed: Property not found. retry in 1.00240275s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:03.2608802Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb\") failed: Property not found. retry in 548.007133ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:03.274350901Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1\") failed: Property not found. retry in 993.464355ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:03.299985688Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\") failed: Property not found. retry in 1.04995281s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:03.320940833Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203\") failed: Property not found. retry in 590.803133ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:03.335523813Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b\") failed: Property not found. retry in 932.881571ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:03.421746566Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660\") failed: Property not found. retry in 554.454378ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:03.441407027Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f\") failed: Property not found. retry in 1.177682477s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:03.626132933Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd\") failed: Property not found. retry in 1.30799087s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:03.639175753Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1\") failed: Property not found. retry in 1.490811262s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:03.656268077Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec\") failed: Property not found. retry in 1.027197631s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:03.667589202Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17\") failed: Property not found. retry in 698.995319ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:03.685814586Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\") failed: Property not found. retry in 1.063966014s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:03.698983159Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957\") failed: Property not found. retry in 831.637317ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:03.73248784Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603\") failed: Property not found. retry in 662.521748ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:03.775225082Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb\") failed: Property not found. retry in 820.005348ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:03.836434345Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7\") failed: Property not found. retry in 1.434552354s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:03.864466973Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203\") failed: Property not found. retry in 745.87054ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:03.939753372Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660\") failed: Property not found. retry in 876.486731ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:04.012085894Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9\") failed: Property not found. retry in 1.162644362s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:04.041036807Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284\") failed: Property not found. retry in 803.296118ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:04.224675795Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b\") failed: Property not found. retry in 1.212643552s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:04.277837648Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1\") failed: Property not found. retry in 920.185484ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:04.306766341Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b\") failed: Property not found. retry in 1.17941535s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:04.369641486Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\") failed: Property not found. retry in 976.850958ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:04.386820033Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17\") failed: Property not found. retry in 973.957083ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:04.400418507Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603\") failed: Property not found. retry in 959.079993ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:04.456687071Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1\") failed: Property not found. retry in 763.724829ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:04.560722508Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957\") failed: Property not found. retry in 654.639514ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:04.578751617Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb\") failed: Property not found. retry in 1.149114407s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:04.670998587Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203\") failed: Property not found. retry in 1.367798545s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:04.700137035Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec\") failed: Property not found. retry in 1.195053839s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:04.714668533Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\") failed: Property not found. retry in 1.466575814s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:04.780844615Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f\") failed: Property not found. retry in 585.588943ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:04.820896147Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660\") failed: Property not found. retry in 1.302055956s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:04.902425298Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd\") failed: Property not found. retry in 1.331348029s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:04.961818573Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284\") failed: Property not found. retry in 1.151995214s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:05.059064693Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1\") failed: Property not found. retry in 1.385555209s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:05.163601783Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9\") failed: Property not found. retry in 1.445896123s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:05.219107457Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1\") failed: Property not found. retry in 1.179287276s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:05.242427144Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957\") failed: Property not found. retry in 1.412339047s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:05.258941453Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7\") failed: Property not found. retry in 783.826625ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:05.311375727Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1\") failed: Property not found. retry in 540.452191ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:05.337294232Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\") failed: Property not found. retry in 568.305554ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:05.378240807Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17\") failed: Property not found. retry in 809.970625ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:05.390735732Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f\") failed: Property not found. retry in 1.092951437s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:05.420333312Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603\") failed: Property not found. retry in 978.591102ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:05.435510857Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b\") failed: Property not found. retry in 837.756052ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:05.504964344Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b\") failed: Property not found. retry in 1.072748893s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:05.563937958Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb\") failed: Property not found. retry in 1.279091631s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:05.834073506Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1\") failed: Property not found. retry in 706.286454ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:05.890104574Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec\") failed: Property not found. retry in 545.952523ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:05.923257087Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\") failed: Property not found. retry in 813.155803ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:05.956274976Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203\") failed: Property not found. retry in 1.011182494s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:06.080390195Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7\") failed: Property not found. retry in 887.609051ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:06.108410714Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17\") failed: Property not found. retry in 658.028116ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:06.211861035Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284\") failed: Property not found. retry in 798.547606ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:06.221608909Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660\") failed: Property not found. retry in 1.424294212s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:06.231702411Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\") failed: Property not found. retry in 866.181632ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:06.259432553Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd\") failed: Property not found. retry in 1.134558603s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:06.305784559Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b\") failed: Property not found. retry in 889.470256ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:06.357228657Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603\") failed: Property not found. retry in 1.19338893s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:06.424721753Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1\") failed: Property not found. retry in 1.493925848s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:06.437561207Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec\") failed: Property not found. retry in 706.910091ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:06.483077782Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f\") failed: Property not found. retry in 534.821142ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:06.524351475Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1\") failed: Property not found. retry in 1.203802686s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:06.561106302Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1\") failed: Property not found. retry in 867.016722ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:06.611182155Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b\") failed: Property not found. retry in 781.667191ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:06.651508494Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9\") failed: Property not found. retry in 977.198666ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:06.677261264Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957\") failed: Property not found. retry in 1.312689384s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:06.68712283Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\") failed: Property not found. retry in 891.817144ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:06.78434026Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17\") failed: Property not found. retry in 753.900992ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:06.882175965Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7\") failed: Property not found. retry in 648.933105ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:07.007779283Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284\") failed: Property not found. retry in 1.29672065s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:07.033391519Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f\") failed: Property not found. retry in 949.879608ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:07.069785466Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203\") failed: Property not found. retry in 614.747368ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:07.104664413Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb\") failed: Property not found. retry in 1.466004798s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:07.124839028Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\") failed: Property not found. retry in 1.116020717s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:07.138592556Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec\") failed: Property not found. retry in 905.399815ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:07.204998204Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b\") failed: Property not found. retry in 915.983867ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:07.261235377Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b\") failed: Property not found. retry in 1.395155264s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:07.44815328Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd\") failed: Property not found. retry in 1.333998406s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:07.459782442Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1\") failed: Property not found. retry in 1.029020056s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:07.49003164Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603\") failed: Property not found. retry in 1.082828029s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:07.631729676Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17\") failed: Property not found. retry in 995.085262ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:07.649444687Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9\") failed: Property not found. retry in 1.137696743s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:07.666027588Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7\") failed: Property not found. retry in 679.481649ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:07.678656267Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660\") failed: Property not found. retry in 900.733372ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:07.703482053Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\") failed: Property not found. retry in 702.161582ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:07.718590546Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203\") failed: Property not found. retry in 1.008025511s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:07.730380833Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1\") failed: Property not found. retry in 1.316947314s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:07.778149425Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1\") failed: Property not found. retry in 1.48015068s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:07.944287777Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957\") failed: Property not found. retry in 1.435641241s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:08.012294887Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f\") failed: Property not found. retry in 1.424308585s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:08.03127283Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec\") failed: Property not found. retry in 623.867117ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:08.123477659Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b\") failed: Property not found. retry in 584.104404ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:08.190096162Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\") failed: Property not found. retry in 560.686859ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:08.267951398Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284\") failed: Property not found. retry in 1.015272536s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:08.342556419Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7\") failed: Property not found. retry in 1.383025486s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:08.371234725Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\") failed: Property not found. retry in 801.617388ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:08.434171872Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1\") failed: Property not found. retry in 1.464208103s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:08.53208698Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb\") failed: Property not found. retry in 933.665929ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:08.606110836Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660\") failed: Property not found. retry in 589.487353ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:08.619214757Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17\") failed: Property not found. retry in 969.865887ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:08.656255081Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603\") failed: Property not found. retry in 1.010621615s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:08.727138375Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203\") failed: Property not found. retry in 1.008160998s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:08.752774062Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec\") failed: Property not found. retry in 1.113585083s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:08.764388784Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b\") failed: Property not found. retry in 1.08864743s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:08.788040129Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd\") failed: Property not found. retry in 1.495547038s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:08.813244505Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9\") failed: Property not found. retry in 1.400957096s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:08.822941527Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\") failed: Property not found. retry in 1.075987383s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:08.847711582Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b\") failed: Property not found. retry in 556.537372ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:08.863679227Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1\") failed: Property not found. retry in 569.123167ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:09.108291041Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660\") failed: Property not found. retry in 610.651086ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:09.221409544Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\") failed: Property not found. retry in 894.271428ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:09.248142769Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284\") failed: Property not found. retry in 912.063346ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:09.368788258Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b\") failed: Property not found. retry in 1.025857981s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:09.430239397Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1\") failed: Property not found. retry in 644.205606ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:09.439515528Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957\") failed: Property not found. retry in 679.108844ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:09.485634788Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f\") failed: Property not found. retry in 692.777912ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:09.494574601Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb\") failed: Property not found. retry in 1.03476986s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:09.549341056Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17\") failed: Property not found. retry in 1.127924977s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:09.635384515Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1\") failed: Property not found. retry in 1.07559805s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:09.687353067Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603\") failed: Property not found. retry in 890.05945ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:09.747720237Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7\") failed: Property not found. retry in 919.193269ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:09.762310847Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203\") failed: Property not found. retry in 1.217117288s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:09.769859963Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660\") failed: Property not found. retry in 1.092358996s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:09.842903074Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b\") failed: Property not found. retry in 967.043175ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:09.888726996Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec\") failed: Property not found. retry in 1.087131737s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:09.899175487Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\") failed: Property not found. retry in 529.434344ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:09.934102646Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1\") failed: Property not found. retry in 1.108730039s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:10.007221958Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1\") failed: Property not found. retry in 1.23335225s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:10.093032291Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\") failed: Property not found. retry in 1.100172911s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:10.150449325Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957\") failed: Property not found. retry in 900.608734ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:10.174056499Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f\") failed: Property not found. retry in 1.165934719s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:10.197008936Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9\") failed: Property not found. retry in 1.431754448s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:10.233204458Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284\") failed: Property not found. retry in 976.684379ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:10.291908055Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd\") failed: Property not found. retry in 1.050944033s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:10.318951749Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b\") failed: Property not found. retry in 1.078131109s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:10.467796621Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\") failed: Property not found. retry in 1.147461742s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:10.478212212Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb\") failed: Property not found. retry in 846.86342ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:10.594173649Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603\") failed: Property not found. retry in 1.224681638s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:10.647696012Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7\") failed: Property not found. retry in 1.028499076s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:10.691334807Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17\") failed: Property not found. retry in 1.229944873s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:10.775345413Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1\") failed: Property not found. retry in 1.330660695s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:10.785555479Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b\") failed: Property not found. retry in 847.929396ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:10.864824571Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660\") failed: Property not found. retry in 1.141466553s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:10.94514986Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec\") failed: Property not found. retry in 696.62597ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:10.995775048Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203\") failed: Property not found. retry in 1.356465877s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:11.003905099Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957\") failed: Property not found. retry in 1.10926155s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:11.08349443Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1\") failed: Property not found. retry in 938.631896ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:11.123510951Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\") failed: Property not found. retry in 1.06026547s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:11.259618352Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284\") failed: Property not found. retry in 1.47368444s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:11.279486179Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1\") failed: Property not found. retry in 659.001009ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:11.335727032Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f\") failed: Property not found. retry in 1.227875635s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:11.371865362Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd\") failed: Property not found. retry in 827.677ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:11.379894231Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb\") failed: Property not found. retry in 961.680138ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:11.449554394Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b\") failed: Property not found. retry in 1.091483399s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:11.554499274Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\") failed: Property not found. retry in 1.250420017s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:11.635101421Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9\") failed: Property not found. retry in 780.054774ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:11.67465709Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec\") failed: Property not found. retry in 1.036939958s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:11.699774034Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b\") failed: Property not found. retry in 754.801684ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:11.720297018Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7\") failed: Property not found. retry in 884.913738ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:11.727492825Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603\") failed: Property not found. retry in 1.181368032s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:11.880586298Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1\") failed: Property not found. retry in 1.234596674s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:12.004686197Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17\") failed: Property not found. retry in 1.239400401s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:12.01328885Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1\") failed: Property not found. retry in 1.0258881s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:12.071833413Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660\") failed: Property not found. retry in 1.172807247s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:12.096209968Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1\") failed: Property not found. retry in 946.939416ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:12.124379931Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957\") failed: Property not found. retry in 568.08243ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:12.2031638Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd\") failed: Property not found. retry in 935.265526ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:12.219085725Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\") failed: Property not found. retry in 1.123041523s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:12.329736643Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203\") failed: Property not found. retry in 742.485142ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:12.368592934Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb\") failed: Property not found. retry in 869.466075ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:12.420836194Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9\") failed: Property not found. retry in 621.783326ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:12.464466189Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b\") failed: Property not found. retry in 1.327329272s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:12.484179492Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f\") failed: Property not found. retry in 556.855942ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:12.608836405Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7\") failed: Property not found. retry in 1.445001468s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:12.619244996Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b\") failed: Property not found. retry in 1.107549634s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:12.65555009Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec\") failed: Property not found. retry in 564.820143ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:12.746992349Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284\") failed: Property not found. retry in 896.92475ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:12.762362329Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957\") failed: Property not found. retry in 555.200717ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:12.780595893Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\") failed: Property not found. retry in 723.947718ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:12.894607269Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603\") failed: Property not found. retry in 781.233115ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:13.073082072Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1\") failed: Property not found. retry in 641.632247ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:13.081538542Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9\") failed: Property not found. retry in 1.4921021s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:13.096293706Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1\") failed: Property not found. retry in 782.654553ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:13.107905098Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203\") failed: Property not found. retry in 1.197434739s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:13.120494076Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd\") failed: Property not found. retry in 573.543571ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:13.163248208Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f\") failed: Property not found. retry in 507.761374ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:13.175770134Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1\") failed: Property not found. retry in 1.287111127s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:13.2478897Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17\") failed: Property not found. retry in 1.403151212s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:13.26211203Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660\") failed: Property not found. retry in 916.980655ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:13.278746183Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb\") failed: Property not found. retry in 843.740343ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:13.299225726Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec\") failed: Property not found. retry in 744.61498ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:13.321205738Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957\") failed: Property not found. retry in 759.647302ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:13.345035708Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\") failed: Property not found. retry in 643.620871ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:13.461635511Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\") failed: Property not found. retry in 770.758975ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:13.628536304Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284\") failed: Property not found. retry in 504.184771ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:13.66799469Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f\") failed: Property not found. retry in 1.180343886s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:13.692993741Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1\") failed: Property not found. retry in 1.265324031s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:13.732663883Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd\") failed: Property not found. retry in 591.569499ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:13.7464126Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b\") failed: Property not found. retry in 667.969785ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:13.773177477Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b\") failed: Property not found. retry in 1.276313819s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:13.820366975Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1\") failed: Property not found. retry in 1.372615146s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:13.938865777Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7\") failed: Property not found. retry in 1.499873103s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:14.073806588Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec\") failed: Property not found. retry in 780.072551ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:14.082187286Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\") failed: Property not found. retry in 1.469253291s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:14.125457902Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb\") failed: Property not found. retry in 1.269962949s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:14.152171737Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284\") failed: Property not found. retry in 990.2464ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:14.182143497Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660\") failed: Property not found. retry in 1.435290517s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:14.21030975Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:14.215212077Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:53433"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:14.284631663Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"58e2bd4c-d4a3-487d-844c-cc68a8159e81","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:53433","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":69206,"response_code":"200"} {"msg":"wait for service Some(\"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203\") failed: Property not found. retry in 1.221630825s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:14.328851784Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd\") failed: Property not found. retry in 914.191553ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:14.347805857Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\") failed: Property not found. retry in 860.215895ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:14.408725342Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b\") failed: Property not found. retry in 986.453614ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:14.450189491Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1\") failed: Property not found. retry in 731.256232ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:14.545226583Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9\") failed: Property not found. retry in 957.799811ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:14.597429351Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f\") failed: Property not found. retry in 1.046983563s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:14.885833125Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b\") failed: Property not found. retry in 1.049930299s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:15.106348912Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd\") failed: Property not found. retry in 522.631368ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:15.270241126Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\") failed: Property not found. retry in 1.014942922s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:15.278723147Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb\") failed: Property not found. retry in 722.923174ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:15.437155248Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b\") failed: Property not found. retry in 765.883383ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:15.447142418Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\") failed: Property not found. retry in 819.94498ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:15.630967461Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f\") failed: Property not found. retry in 1.414666449s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:15.94455772Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b\") failed: Property not found. retry in 1.253509313s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:16.16635037Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:17.498216661Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_crucible_bd7f3d92-4497-48dd-9f51-160ce4d7f4d9"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:17.498287023Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_crucible_c6d325c3-cddf-4528-8ba6-a083f6511284"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:17.498295043Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_crucible_c30b7bc1-fd10-4942-8055-ada48d2e3957"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:17.498300223Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_external_dns_1eb90f9c-ba99-428e-885b-5fa2df7ae2f1"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:17.498311524Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_nexus_742e0330-77de-4723-a145-74751d41c73b"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:17.498316614Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_crucible_pantry_3286a4ca-fffd-4ed0-9c21-d528e1d23603"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:17.498322154Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_crucible_6fa8dbd2-a741-440b-98a2-b60bf983cdc7"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:17.498327384Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_clickhouse_2212adda-2fcf-4d59-888c-2223a4fe3f6b"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:17.498332434Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_crucible_1bd16d4a-6b16-4df4-b218-9e6f3ad1d9fb"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:17.498337574Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_crucible_139ec2b6-5f74-4c07-9d4b-6e2f8dbeab17"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:17.498345314Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_nexus_87d65a87-e1e5-43e4-9cba-b4afd5e9ac52"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:17.498350265Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_nexus_6088410e-a8fc-4d33-b593-6385e863291f"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:17.498354625Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_oximeter_8d644923-5967-4351-a1ad-fe8a938f33e1"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:17.498361015Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_crucible_pantry_64cdbe7d-ce11-4095-b7f7-bf51bfb3f7ec"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:17.498366155Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_crucible_pantry_740bd3e0-b30c-4033-968a-5451c8d5adcd"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:17.498382946Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_crucible_0b1925c8-3ae5-4372-a3de-213551574fe1"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:17.498388036Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_crucible_663bd1f7-7a29-484c-bfce-d312646b1660"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:17.498392356Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_crucible_605558ae-0f1e-47a7-b88c-f397548d4ea1"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:17.498397056Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2980","zone":"oxz_external_dns_09fdc6fd-8297-436e-903d-b18710599203"} {"msg":"Writing ledger to /pool/int/301424fa-c483-4ce9-92fb-3c8ee10e6f9f/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:17.51315849Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/20dccfdd-e68a-468d-8629-7f7e0834aa1c/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:17.514614048Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:17.529250869Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"a8598c33-c3b0-4fed-943d-33cf06836681","remote_addr":"[fd00:1122:3344:101::1]:37752","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":29065749,"response_code":"204"} {"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:17.618966883Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1244"} {"msg":"Writing ledger to /pool/int/301424fa-c483-4ce9-92fb-3c8ee10e6f9f/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:17.618997653Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/20dccfdd-e68a-468d-8629-7f7e0834aa1c/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:17.619157348Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:17.623557332Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:681"} {"msg":"Nexus address: [fd00:1122:3344:101::b]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:17.647734771Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:694"} {"msg":"rack_network_config: RackNetworkConfigV2 {\n bfd: [],\n bgp: [],\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV2 {\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.1.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n autoneg: false,\n bgp_peers: [],\n port: \"qsfp0\",\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.1.100,\n vlan_id: None,\n },\n ],\n switch: Switch0,\n uplink_port_fec: None,\n uplink_port_speed: Speed40G,\n },\n ],\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:17.7618223Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:825"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:47:20.876180025Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/metrics/producers): error trying to connect: tcp connect error: Connection refused (os error 146)\"","delay":"89.091825153s"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/0ac29522-8a9f-4432-bdfe-a8adde17b3b5/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:20.876436882Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/0ac29522-8a9f-4432-bdfe-a8adde17b3b5/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:20.977311166Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/0ac29522-8a9f-4432-bdfe-a8adde17b3b5/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:21.100713557Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/0ac29522-8a9f-4432-bdfe-a8adde17b3b5/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:21.277080195Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/0ac29522-8a9f-4432-bdfe-a8adde17b3b5/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:21.377555999Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/0ac29522-8a9f-4432-bdfe-a8adde17b3b5/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:21.655316186Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/0ac29522-8a9f-4432-bdfe-a8adde17b3b5/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:21.805589696Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/0ac29522-8a9f-4432-bdfe-a8adde17b3b5/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:22.065008685Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/0ac29522-8a9f-4432-bdfe-a8adde17b3b5/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:22.315501382Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/0ac29522-8a9f-4432-bdfe-a8adde17b3b5/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:22.841057015Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/0ac29522-8a9f-4432-bdfe-a8adde17b3b5/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:23.192484528Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/0ac29522-8a9f-4432-bdfe-a8adde17b3b5/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:23.685831343Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/0ac29522-8a9f-4432-bdfe-a8adde17b3b5/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:24.5555379Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/0ac29522-8a9f-4432-bdfe-a8adde17b3b5/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:25.340138823Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/0ac29522-8a9f-4432-bdfe-a8adde17b3b5/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:26.018619455Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/0ac29522-8a9f-4432-bdfe-a8adde17b3b5/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:27.349701086Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/0ac29522-8a9f-4432-bdfe-a8adde17b3b5/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:29.298190958Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Error Response: status: 403 Forbidden; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"eaadb9e4-0410-44d9-9ac4-6143382572d6\", \"content-length\": \"113\", \"date\": \"Fri, 19 Jul 2024 17:47:30 GMT\"}; value: Error { error_code: Some(\"Forbidden\"), message: \"Forbidden\", request_id: \"eaadb9e4-0410-44d9-9ac4-6143382572d6\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:31.185256103Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ba92bfcd-3ffe-4a1a-bf14-9418c6f5471b\", \"content-length\": \"173\", \"date\": \"Fri, 19 Jul 2024 17:47:33 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"0c1108fb-cc91-4535-adf7-846d50ef8716\\\"\", request_id: \"ba92bfcd-3ffe-4a1a-bf14-9418c6f5471b\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:34.324056933Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"73264092-d8e0-426e-b2f5-385261646d70\", \"content-length\": \"173\", \"date\": \"Fri, 19 Jul 2024 17:47:36 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"0c1108fb-cc91-4535-adf7-846d50ef8716\\\"\", request_id: \"73264092-d8e0-426e-b2f5-385261646d70\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:36.071814923Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0f4ca73b-94f3-4cfa-9dc7-274c3c32fc2e\", \"content-length\": \"173\", \"date\": \"Fri, 19 Jul 2024 17:47:39 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"0c1108fb-cc91-4535-adf7-846d50ef8716\\\"\", request_id: \"0f4ca73b-94f3-4cfa-9dc7-274c3c32fc2e\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:39.855331066Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:44.322490105Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:51922"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:44.322787643Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"884e6751-75c0-499f-8b11-f6d0f242f037","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:51922","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":38,"response_code":"200"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"633313ed-9e18-45ad-a875-7ec65f7b5549\", \"content-length\": \"173\", \"date\": \"Fri, 19 Jul 2024 17:47:44 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"0c1108fb-cc91-4535-adf7-846d50ef8716\\\"\", request_id: \"633313ed-9e18-45ad-a875-7ec65f7b5549\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:44.343708847Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9572ae4d-01cb-4300-862c-0851bce7d8e8\", \"content-length\": \"173\", \"date\": \"Fri, 19 Jul 2024 17:47:51 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"0c1108fb-cc91-4535-adf7-846d50ef8716\\\"\", request_id: \"9572ae4d-01cb-4300-862c-0851bce7d8e8\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:51.379535554Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4219d8e3-8b7c-4595-89b5-352da8999381\", \"content-length\": \"173\", \"date\": \"Fri, 19 Jul 2024 17:47:55 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"0c1108fb-cc91-4535-adf7-846d50ef8716\\\"\", request_id: \"4219d8e3-8b7c-4595-89b5-352da8999381\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:55.307177728Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Successfully put SledAgentInfo to nexus","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:58.291096518Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"sled-agent/src/nexus.rs:592","new_generation":"1","old_generation":"1"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:58.421703656Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:34400"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:58.425259419Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:58.512403916Z","hostname":"buskin","pid":1555,"uri":"/inventory","method":"GET","req_id":"0b4fae87-f976-4130-80d8-34c2cc51170e","remote_addr":"[fd00:1122:3344:101::c]:34400","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":89540,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:58.514385297Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/301424fa-c483-4ce9-92fb-3c8ee10e6f9f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:58.514669095Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/20dccfdd-e68a-468d-8629-7f7e0834aa1c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:58.520790324Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:47:58.521375089Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"GET","req_id":"8382a9b4-52e2-4115-87e2-011395352603","remote_addr":"[fd00:1122:3344:101::c]:34400","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":7234,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:01.774453413Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:37545"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:01.774631888Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"9ff43996-de9a-419d-b71b-4b0cad3a4bf9","remote_addr":"[fd00:1122:3344:101::c]:37545","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":129,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:01.859339351Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"094dc056-ea38-4a54-ae31-60f487ec1840","remote_addr":"[fd00:1122:3344:101::c]:37545","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":8923,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:01.865343947Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:37450"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:01.865678696Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"335c306a-3a60-4ef0-929e-2ac8f14b5f02","remote_addr":"[fd00:1122:3344:101::c]:37450","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":280,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:01.946216761Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:39199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:01.946857368Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"25cec175-cb10-4ab6-b289-3f365d044509","remote_addr":"[fd00:1122:3344:101::a]:39199","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":591,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:01.950067441Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:37624"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:01.950203565Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"cd460589-0067-4a26-851d-fcb93bcc228f","remote_addr":"[fd00:1122:3344:101::a]:37624","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":100,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:01.953774348Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"713ac1a5-9369-4faa-b386-bd08de04d244","remote_addr":"[fd00:1122:3344:101::a]:39199","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":72,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:02.583271866Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:65207"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:02.583509362Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"a4fedbcb-3f61-4b68-91fd-c9b3f9aa6be7","remote_addr":"[fd00:1122:3344:101::b]:65207","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":217,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:02.656062649Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:51787"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:02.656165282Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"bc98ed30-392c-4c16-9062-499f2863bfad","remote_addr":"[fd00:1122:3344:101::b]:51787","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":68,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:02.662810335Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"74f32ab7-e650-46e3-accb-5531d1063799","remote_addr":"[fd00:1122:3344:101::b]:51787","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":130,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:05.122111146Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:48065"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:05.125922745Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:673","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 005b2740-84d5-473e-ae59-aa79fd2fe5b6, kind: Service { id: 09fdc6fd-8297-436e-903d-b18710599203 }, name: Name(\"external-dns-09fdc6fd-8297-436e-903d-b18710599203\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 59731bd7-ffc9-493f-a33c-a1cf5ba2df30, kind: Service { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1 }, name: Name(\"external-dns-1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 32])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: c6edf68c-f558-4f51-9aed-6eb91fb3ad51, kind: Service { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52 }, name: Name(\"nexus-87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 117])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e6898038-bf11-41c8-b971-6ff3eed6d38b, kind: Service { id: 6088410e-a8fc-4d33-b593-6385e863291f }, name: Name(\"nexus-6088410e-a8fc-4d33-b593-6385e863291f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 27])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ffe6d41d-1349-42b2-a887-49d734eef422, kind: Service { id: 742e0330-77de-4723-a145-74751d41c73b }, name: Name(\"nexus-742e0330-77de-4723-a145-74751d41c73b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:05.126684925Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:05.127373183Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:05.127391004Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:05.127397364Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:05.127416324Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:05.127432705Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:05.127463376Z","hostname":"buskin","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"d3a78ea5-b9af-475e-af09-bd0c607630eb","remote_addr":"[fd00:1122:3344:101::b]:48065","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":5275,"response_code":"204"} {"msg":"Handoff to Nexus is complete","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:05.129087638Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:897"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:05.17875976Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:63841"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:05.179173151Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:673","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 005b2740-84d5-473e-ae59-aa79fd2fe5b6, kind: Service { id: 09fdc6fd-8297-436e-903d-b18710599203 }, name: Name(\"external-dns-09fdc6fd-8297-436e-903d-b18710599203\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 59731bd7-ffc9-493f-a33c-a1cf5ba2df30, kind: Service { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1 }, name: Name(\"external-dns-1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 32])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: c6edf68c-f558-4f51-9aed-6eb91fb3ad51, kind: Service { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52 }, name: Name(\"nexus-87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 117])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e6898038-bf11-41c8-b971-6ff3eed6d38b, kind: Service { id: 6088410e-a8fc-4d33-b593-6385e863291f }, name: Name(\"nexus-6088410e-a8fc-4d33-b593-6385e863291f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 27])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ffe6d41d-1349-42b2-a887-49d734eef422, kind: Service { id: 742e0330-77de-4723-a145-74751d41c73b }, name: Name(\"nexus-742e0330-77de-4723-a145-74751d41c73b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:05.179232192Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:05.179280144Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:05.179293614Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:05.179301504Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:05.179318895Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:05.179336745Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:05.179355966Z","hostname":"buskin","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"c1a547f0-1063-4d70-8210-89a152e20d30","remote_addr":"[fd00:1122:3344:101::c]:63841","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":350,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:05.180076144Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:44708"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:05.180548107Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:673","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 005b2740-84d5-473e-ae59-aa79fd2fe5b6, kind: Service { id: 09fdc6fd-8297-436e-903d-b18710599203 }, name: Name(\"external-dns-09fdc6fd-8297-436e-903d-b18710599203\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 59731bd7-ffc9-493f-a33c-a1cf5ba2df30, kind: Service { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1 }, name: Name(\"external-dns-1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 32])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: c6edf68c-f558-4f51-9aed-6eb91fb3ad51, kind: Service { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52 }, name: Name(\"nexus-87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 117])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e6898038-bf11-41c8-b971-6ff3eed6d38b, kind: Service { id: 6088410e-a8fc-4d33-b593-6385e863291f }, name: Name(\"nexus-6088410e-a8fc-4d33-b593-6385e863291f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 27])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ffe6d41d-1349-42b2-a887-49d734eef422, kind: Service { id: 742e0330-77de-4723-a145-74751d41c73b }, name: Name(\"nexus-742e0330-77de-4723-a145-74751d41c73b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:05.180588818Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:05.180618768Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:05.180629519Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:05.180649649Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:05.18068192Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:05.180700061Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:05.180716401Z","hostname":"buskin","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"a2a37b9e-3ae5-472a-8da1-c1a7accb085e","remote_addr":"[fd00:1122:3344:101::b]:44708","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":566,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:05.208909104Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:47926"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:05.20910923Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:673","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 005b2740-84d5-473e-ae59-aa79fd2fe5b6, kind: Service { id: 09fdc6fd-8297-436e-903d-b18710599203 }, name: Name(\"external-dns-09fdc6fd-8297-436e-903d-b18710599203\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 59731bd7-ffc9-493f-a33c-a1cf5ba2df30, kind: Service { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1 }, name: Name(\"external-dns-1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 32])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: c6edf68c-f558-4f51-9aed-6eb91fb3ad51, kind: Service { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52 }, name: Name(\"nexus-87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 117])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e6898038-bf11-41c8-b971-6ff3eed6d38b, kind: Service { id: 6088410e-a8fc-4d33-b593-6385e863291f }, name: Name(\"nexus-6088410e-a8fc-4d33-b593-6385e863291f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 27])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ffe6d41d-1349-42b2-a887-49d734eef422, kind: Service { id: 742e0330-77de-4723-a145-74751d41c73b }, name: Name(\"nexus-742e0330-77de-4723-a145-74751d41c73b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:05.209144061Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:05.209163781Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:05.209173881Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:05.209202062Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:05.209225673Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:05.209242803Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:05.209258344Z","hostname":"buskin","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"c66c2e62-678e-4af5-9cd4-1d507baec81d","remote_addr":"[fd00:1122:3344:101::a]:47926","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":250,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:05.381012892Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:57723"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:05.382056149Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:05.47318406Z","hostname":"buskin","pid":1555,"uri":"/inventory","method":"GET","req_id":"a429c85c-bfa5-4e86-9671-50475d8929d6","remote_addr":"[fd00:1122:3344:101::b]:57723","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":92076,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:05.47357826Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/301424fa-c483-4ce9-92fb-3c8ee10e6f9f/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:05.473651592Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/20dccfdd-e68a-468d-8629-7f7e0834aa1c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:05.474180096Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:05.474893115Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"GET","req_id":"c56d0820-50a1-4d26-9b40-9c259392eac1","remote_addr":"[fd00:1122:3344:101::b]:57723","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":1421,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:10.155468026Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:39577"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:10.155609959Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"289f51d1-3b2e-4356-bf0b-ac0feb39de18","remote_addr":"[fd00:1122:3344:101::b]:39577","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":121,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:10.161366409Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"456702e3-7d54-4f01-87d4-fb65ea6c466f","remote_addr":"[fd00:1122:3344:101::b]:39577","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":131,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:14.361063769Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:40047"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:14.361732346Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"adfbf536-ad08-4a95-945f-53152c55b499","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:40047","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":629,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:31.834095742Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:59044"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:31.834276046Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"909343c7-f770-4b7b-8da3-2407d8424e80","remote_addr":"[fd00:1122:3344:101::a]:59044","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":155,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:31.842527531Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"dfe045f6-0ce5-4a0f-86b5-0ee47bf579ed","remote_addr":"[fd00:1122:3344:101::a]:59044","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":138,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:32.081536958Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:34217"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:32.082313919Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"73f4c82b-57d4-474c-bd61-388cf171fe3c","remote_addr":"[fd00:1122:3344:101::c]:34217","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":746,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:32.083736016Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:61018"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:32.083891509Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"5de6a38f-204f-466a-aadc-9ab4825400c5","remote_addr":"[fd00:1122:3344:101::a]:61018","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":135,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:32.089380202Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"756db38c-cee9-44a5-8763-8b9d1644d26c","remote_addr":"[fd00:1122:3344:101::c]:34217","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":84,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:32.148640924Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:36558"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:32.149429905Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"14434cfc-2264-485f-bcdf-a034a2b3c066","remote_addr":"[fd00:1122:3344:101::c]:36558","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":742,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:32.333298348Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:46988"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:32.418075824Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"ffd15340-83f0-4393-83c7-243114cf2b08","remote_addr":"[fd00:1122:3344:101::a]:46988","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":84707,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:32.419535102Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"4d3d5da6-3b91-4b18-a9f8-51393a155d35","remote_addr":"[fd00:1122:3344:101::a]:46988","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":1232,"response_code":"200"} {"msg":"Attempting to update network config with generation: 2, current_generation: 1","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:32.442114749Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/301424fa-c483-4ce9-92fb-3c8ee10e6f9f/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:32.442995592Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/20dccfdd-e68a-468d-8629-7f7e0834aa1c/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:32.444711297Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 2","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:32.446526514Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:32.446552055Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"PUT","req_id":"31309138-162e-4d8f-9dce-7e4b65a1a34a","remote_addr":"[fd00:1122:3344:101::a]:46988","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":6921,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:32.447568801Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:46620"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:32.508200549Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"c49be600-41d1-40c9-abd2-7a8ea640fdd2","remote_addr":"[fd00:1122:3344:101::c]:46620","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":60548,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:32.511375681Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"b3ca0fbe-235c-4793-b7ec-8c853d12b653","remote_addr":"[fd00:1122:3344:101::c]:46620","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":846,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:32.930727091Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:35675"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:32.931682726Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"edac0aec-e2ef-48a9-870c-16e555bcc5e5","remote_addr":"[fd00:1122:3344:101::b]:35675","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":926,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:32.948782351Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:65195"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:32.948881484Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"b05ec78b-98f8-413e-bd1a-830e53673e5c","remote_addr":"[fd00:1122:3344:101::b]:65195","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":53,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:32.956078171Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"12ec588f-525e-4ec3-8c6f-bfc7b338c647","remote_addr":"[fd00:1122:3344:101::b]:65195","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":98,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:33.043029483Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:45639"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:33.093240299Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"888e05e2-1c71-46c4-9674-66454a1c6ab4","remote_addr":"[fd00:1122:3344:101::b]:45639","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":50147,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:33.096595776Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"4e226918-16ff-438d-b1b0-4bcbf6fb1f95","remote_addr":"[fd00:1122:3344:101::b]:45639","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":815,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:44.39542783Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:45020"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:48:44.395915023Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"0d2e7e58-41a9-42d9-b79e-81ea0186e7cf","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:45020","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":470,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:01.851731339Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:64089"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:01.851962565Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"b63f95fd-0e0b-42dd-a476-40f1ed968db9","remote_addr":"[fd00:1122:3344:101::a]:64089","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":209,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:01.895359744Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:50650"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:01.895442396Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"1d8aaebb-d230-47cb-be51-35ca2d76f7a8","remote_addr":"[fd00:1122:3344:101::a]:50650","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":70,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:01.903426644Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"6ba68314-4fc6-4848-bc0b-57a5ca47f35b","remote_addr":"[fd00:1122:3344:101::a]:50650","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":113,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:01.951416743Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:60800"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:01.952219924Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"959407d0-ee69-48a7-98ed-ce4add78016e","remote_addr":"[fd00:1122:3344:101::c]:60800","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":759,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:01.955345185Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:55661"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:01.955412156Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"ce73ea46-82e5-4d64-a2d9-305cece0dcf0","remote_addr":"[fd00:1122:3344:101::c]:55661","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":24,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:01.962636415Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"31ab91aa-609e-474d-85ef-154a1d0bbf9e","remote_addr":"[fd00:1122:3344:101::c]:55661","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":55,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:01.988483737Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:60361"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:02.035709045Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"85b8e6a8-c79d-4baf-b6e9-b20a6aea639b","remote_addr":"[fd00:1122:3344:101::a]:60361","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":47180,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:02.035995163Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"e5bf7201-f353-45fe-8d7d-d0561cb168d1","remote_addr":"[fd00:1122:3344:101::a]:60361","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":86,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:02.065690496Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:33431"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:02.111295372Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"a58d5e5a-451a-482e-9095-39754e840288","remote_addr":"[fd00:1122:3344:101::c]:33431","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":45515,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:02.111573439Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"285042f9-c52d-4302-8370-c2b38452d6a9","remote_addr":"[fd00:1122:3344:101::c]:33431","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":103,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:02.605748696Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:33411"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:02.606026573Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"e5df0bcf-d7b7-4c21-87cf-708ec5b4c741","remote_addr":"[fd00:1122:3344:101::b]:33411","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":219,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:02.738015447Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:49041"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:02.7381297Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"fad4eac7-16fd-4489-bb45-c2ebcf6f7abf","remote_addr":"[fd00:1122:3344:101::b]:49041","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":81,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:02.745658656Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"056da02f-78d2-4d5c-82c3-8c11767f23eb","remote_addr":"[fd00:1122:3344:101::b]:49041","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":101,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:02.82846805Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:65512"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:02.875537285Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"7fc059e9-806e-4169-825c-58bd30e33944","remote_addr":"[fd00:1122:3344:101::b]:65512","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":47030,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:02.875811972Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"a7c9141c-ed5a-4591-8c5e-c9ac95916210","remote_addr":"[fd00:1122:3344:101::b]:65512","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":106,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:14.428178682Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:40930"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:14.428304606Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"8fbecb73-80e7-4cf9-841a-5843919d6ab4","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:40930","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":95,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:20.115115304Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:56984","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","component":"producer-server","component":"MetricsManager","sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::d]:57512"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:20.127788744Z","hostname":"buskin","pid":1555,"uri":"/0c1108fb-cc91-4535-adf7-846d50ef8716","method":"GET","req_id":"dfbeeb30-5a37-4cb3-8118-aa134b5aa293","remote_addr":"[fd00:1122:3344:101::d]:57512","local_addr":"[fd00:1122:3344:101::1]:56984","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":9058,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:31.516777504Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:31.928365472Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:51195"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:31.928459675Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"76e7a104-db4c-4a51-bdf9-28664329e323","remote_addr":"[fd00:1122:3344:101::c]:51195","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":71,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:31.936313959Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"84fc0552-3919-4d7d-85b0-b6f9b6c769f1","remote_addr":"[fd00:1122:3344:101::c]:51195","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":117,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:31.963466295Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:34046"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:31.963613229Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"c6bbe0bf-b338-40c9-a230-f8d9b6e00242","remote_addr":"[fd00:1122:3344:101::c]:34046","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":116,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:31.969280017Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:36894"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:31.970086188Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"a355ef69-f7df-4664-b9d7-43c8d11b17c4","remote_addr":"[fd00:1122:3344:101::a]:36894","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":766,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:32.05052671Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:42701"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:32.050604662Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"0440cab7-639c-463e-bb5d-f64ba8123093","remote_addr":"[fd00:1122:3344:101::a]:42701","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":35,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:32.059958896Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"37615928-4f6b-478c-b1f4-04ddd305ab93","remote_addr":"[fd00:1122:3344:101::a]:42701","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":87,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:32.222684569Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:46455"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:32.273024949Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"92263a3f-12a8-40dc-a446-024c54efa8cd","remote_addr":"[fd00:1122:3344:101::a]:46455","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":50292,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:32.273146372Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:60784"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:32.276597342Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"bd2590ce-9754-4a86-8e07-e4b9acf37dae","remote_addr":"[fd00:1122:3344:101::a]:46455","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":3219,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:32.320627427Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"6e34021c-50a8-48f1-96aa-e0668f303d1f","remote_addr":"[fd00:1122:3344:101::c]:60784","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":47389,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:32.321670745Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"d8f60fb5-a2bf-490b-842f-71c142445ef4","remote_addr":"[fd00:1122:3344:101::c]:60784","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":801,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:32.618563619Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:54325"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:32.618664112Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"dffaffff-66f1-487e-ae43-36cb1aadf188","remote_addr":"[fd00:1122:3344:101::b]:54325","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":66,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:32.627193113Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"7d85a663-598f-4d5e-a6cc-862aab1fcaae","remote_addr":"[fd00:1122:3344:101::b]:54325","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":115,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:32.718566291Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:63715"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:32.718745125Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"a7eeb8fb-bbaa-499b-ac30-4bc7d618b928","remote_addr":"[fd00:1122:3344:101::b]:63715","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":139,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:32.935278709Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:49458"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:32.983532094Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"4b80fa45-1c26-44e1-b8e1-58a244d3cc3a","remote_addr":"[fd00:1122:3344:101::b]:49458","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":48159,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:32.983983896Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"f251e753-1018-4a45-b515-74fc7a38e367","remote_addr":"[fd00:1122:3344:101::b]:49458","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":77,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:44.462180286Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:53010"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:44.4622949Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"231d75ad-4916-4cc6-bda9-92b6f1206af1","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:53010","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":118,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:49:50.114407987Z","hostname":"buskin","pid":1555,"uri":"/0c1108fb-cc91-4535-adf7-846d50ef8716","method":"GET","req_id":"43ca6d3c-15a9-4a2d-aeaf-a7d44c1b89b8","remote_addr":"[fd00:1122:3344:101::d]:57512","local_addr":"[fd00:1122:3344:101::1]:56984","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":1093,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:01.814403558Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:35483"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:01.815344153Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"a5287d41-3278-4f42-aa53-38c37951d6b0","remote_addr":"[fd00:1122:3344:101::a]:35483","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":925,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:01.815784034Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:60930"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:01.815843806Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"4ddfd068-9dd5-45c1-815b-7e95b9cbda03","remote_addr":"[fd00:1122:3344:101::c]:60930","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":40,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:01.823423153Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"f2cc2ee5-5639-43b8-848f-2f53d37d85ba","remote_addr":"[fd00:1122:3344:101::a]:35483","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":124,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:01.823634298Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"0c58c2f3-17fc-475f-9f4d-8afa742a202d","remote_addr":"[fd00:1122:3344:101::c]:60930","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":43,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:01.846136844Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:38189"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:01.846289408Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"deaa4e22-c86d-4a71-b658-447950c7dbb8","remote_addr":"[fd00:1122:3344:101::a]:38189","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":130,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:01.851444292Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:55149"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:01.851557555Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"2b340d35-d029-4637-93dd-a98a7f1b9be6","remote_addr":"[fd00:1122:3344:101::c]:55149","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":72,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:01.962985794Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:62960"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:02.012620645Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"db636788-1bc1-4d17-8f59-f90df23ef533","remote_addr":"[fd00:1122:3344:101::c]:62960","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":49593,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:02.013504128Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"50ec5868-b0c4-41cc-bc94-05c516376c79","remote_addr":"[fd00:1122:3344:101::c]:62960","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":721,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:02.050375367Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:33239"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:02.096170309Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"30a56b02-1ceb-4338-a263-9f31a03b77a5","remote_addr":"[fd00:1122:3344:101::a]:33239","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":45713,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:02.096417235Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"93ee3637-9ab1-4dec-b7f3-f5a401519cc9","remote_addr":"[fd00:1122:3344:101::a]:33239","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":99,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:02.58822254Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:60539"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:02.588344474Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"0ac852a9-a599-40b6-8709-a8534c90e087","remote_addr":"[fd00:1122:3344:101::b]:60539","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":108,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:02.596517636Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"df69b1a0-0aea-4171-a443-fc984acaf770","remote_addr":"[fd00:1122:3344:101::b]:60539","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":120,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:02.687220966Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:62468"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:02.68738005Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"9f3db2dc-76af-461d-b381-c52df7b8b3c6","remote_addr":"[fd00:1122:3344:101::b]:62468","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":137,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:02.792425503Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:59832"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:02.839298863Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"1787f258-58e2-4b24-961b-3dda78ca857f","remote_addr":"[fd00:1122:3344:101::b]:59832","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":46825,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:02.839564309Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"fd8b856a-a566-447d-92e1-0d8c7b443e5f","remote_addr":"[fd00:1122:3344:101::b]:59832","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":107,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:14.494902468Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:39981"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:14.495482594Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"9afb9039-2ad7-4752-bba8-a9a2d6126947","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:39981","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":549,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:20.115126216Z","hostname":"buskin","pid":1555,"uri":"/0c1108fb-cc91-4535-adf7-846d50ef8716","method":"GET","req_id":"af79bea8-7e77-4b9f-815a-3f8fc69af81f","remote_addr":"[fd00:1122:3344:101::d]:57512","local_addr":"[fd00:1122:3344:101::1]:56984","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":651,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:32.035039158Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:33959"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:32.035244283Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"91af9910-0c88-4da9-938f-6b7c3aacb1b6","remote_addr":"[fd00:1122:3344:101::c]:33959","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":180,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:32.062188854Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:65308"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:32.063243611Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"966c980b-3473-49be-8422-67671abccc06","remote_addr":"[fd00:1122:3344:101::c]:65308","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":1020,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:32.072331538Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"f0309646-b750-48d5-81ec-405042063c59","remote_addr":"[fd00:1122:3344:101::c]:65308","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":119,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:32.094141775Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:33643"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:32.094294309Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"25f1da67-fb87-4839-a70c-a658f128407e","remote_addr":"[fd00:1122:3344:101::a]:33643","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":90,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:32.148581692Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:34000"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:32.148644633Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"dffe87eb-0e6c-492a-8a53-fc0b7803f251","remote_addr":"[fd00:1122:3344:101::a]:34000","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":43,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:32.155980274Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"b663929f-1851-4e61-ab4e-e02a7d9791b8","remote_addr":"[fd00:1122:3344:101::a]:34000","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":79,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:32.320905205Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:38135"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:32.369263613Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"1aee49a8-ac4c-40bd-80f6-887a0abf54fc","remote_addr":"[fd00:1122:3344:101::a]:38135","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":48294,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:32.369356916Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:45921"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:32.415848885Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"c4d334af-975d-42a3-a1ff-e4fdff2068a4","remote_addr":"[fd00:1122:3344:101::c]:45921","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":46427,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:32.416087662Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"7e650900-f07b-4406-9116-7ecf2b91b078","remote_addr":"[fd00:1122:3344:101::a]:38135","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":119,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:32.417079937Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"0cc4c66e-9093-4fa4-9e6c-75d5ea51e94a","remote_addr":"[fd00:1122:3344:101::c]:45921","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":884,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:32.68009215Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:55645"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:32.680198273Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"43af3bce-63f7-40b3-9145-e953f3fae620","remote_addr":"[fd00:1122:3344:101::b]:55645","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":71,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:32.688523459Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"0c60b911-63da-47a1-b426-99a65d1a1300","remote_addr":"[fd00:1122:3344:101::b]:55645","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":891,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:32.849143218Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:52710"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:32.849287492Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"f37be28f-bb74-44ed-a084-fb2be1dea11d","remote_addr":"[fd00:1122:3344:101::b]:52710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":124,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:33.060181539Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:59948"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:33.108574698Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"62fb8251-4980-4852-b2e6-7455b4b7d526","remote_addr":"[fd00:1122:3344:101::b]:59948","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":48329,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:33.108813934Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"3621962d-8cb2-468e-afbe-301092d857d2","remote_addr":"[fd00:1122:3344:101::b]:59948","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":94,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:44.527845067Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:59582"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:44.528473563Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"31908f3c-16e8-47ab-a938-2c07bfa5c484","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:59582","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":622,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:50:50.115448059Z","hostname":"buskin","pid":1555,"uri":"/0c1108fb-cc91-4535-adf7-846d50ef8716","method":"GET","req_id":"9d4ce8c8-64cb-4396-ae3b-62cf68934958","remote_addr":"[fd00:1122:3344:101::d]:57512","local_addr":"[fd00:1122:3344:101::1]:56984","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":1049,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:01.82319159Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:48783"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:01.823344994Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"09946601-d7f4-4266-9437-e6f68b4b586e","remote_addr":"[fd00:1122:3344:101::c]:48783","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":109,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:01.83127122Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"120580a5-d135-477e-82df-d6e677ef2fca","remote_addr":"[fd00:1122:3344:101::c]:48783","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":133,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:01.905869131Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:33274"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:01.906729233Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"307ae6b7-f4d1-43e3-887e-c7828fb10c67","remote_addr":"[fd00:1122:3344:101::a]:33274","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":837,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:01.914264669Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"b7384a57-9bf3-4a30-add5-3aa4807c4c4c","remote_addr":"[fd00:1122:3344:101::a]:33274","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":98,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:01.939961758Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:34411"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:01.940098901Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"72ca5eb3-3992-47c5-8b1a-9034fc7b66ed","remote_addr":"[fd00:1122:3344:101::a]:34411","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":121,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:01.944803104Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:37324"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:01.944925447Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"da8bd239-4365-401d-be8b-0af629143dcc","remote_addr":"[fd00:1122:3344:101::c]:37324","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":67,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:02.019832046Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:44816"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:02.065265478Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"7904554f-4321-43e2-bdc9-62f9a7205cf2","remote_addr":"[fd00:1122:3344:101::a]:44816","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":45395,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:02.06613286Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"b2637611-685c-4e00-9119-cfc646abd73b","remote_addr":"[fd00:1122:3344:101::a]:44816","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":661,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:02.102865576Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:49303"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:02.1448998Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"b5ce7420-de0f-4c45-8b2c-11e6c674c437","remote_addr":"[fd00:1122:3344:101::c]:49303","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":41939,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:02.145594368Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"75742359-7ff2-4db8-901c-1e29fd287db7","remote_addr":"[fd00:1122:3344:101::c]:49303","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":542,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:02.648623475Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:46883"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:02.649632342Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"9022e016-098d-4ad6-91a3-5e803975f55d","remote_addr":"[fd00:1122:3344:101::b]:46883","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":965,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:02.775395934Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:34215"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:02.775490876Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"b8f0491d-9159-4932-a654-7a4b8c7feb61","remote_addr":"[fd00:1122:3344:101::b]:34215","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":72,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:02.788742181Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"237df109-37be-4e02-b0a3-c8a6b405458c","remote_addr":"[fd00:1122:3344:101::b]:34215","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":138,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:02.955246052Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:33145"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:03.001410583Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"b860776f-4037-4ccd-9b1c-e6df8296e5ee","remote_addr":"[fd00:1122:3344:101::b]:33145","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":46121,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:03.002298586Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"bdbafc40-c9a1-46ce-9a30-fba7ca0c3bff","remote_addr":"[fd00:1122:3344:101::b]:33145","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":732,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:14.560954468Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:46393"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:14.561075242Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"6b303ecb-b0da-4c21-b9df-94c142cf98a4","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:46393","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":83,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:20.142204475Z","hostname":"buskin","pid":1555,"uri":"/0c1108fb-cc91-4535-adf7-846d50ef8716","method":"GET","req_id":"4a382bdc-2bab-498a-a576-f749b16bee75","remote_addr":"[fd00:1122:3344:101::d]:57512","local_addr":"[fd00:1122:3344:101::1]:56984","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":7742,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:31.888812098Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:59640"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:31.88889475Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"d9f48c3f-4fff-4c98-8b1b-b324f7cfe692","remote_addr":"[fd00:1122:3344:101::c]:59640","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":69,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:31.896026195Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"0a75f4e4-638c-4d39-bcbb-a0ebc8fc6e08","remote_addr":"[fd00:1122:3344:101::c]:59640","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":123,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:31.928634554Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:42830"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:31.928791718Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"c39d62ab-e5de-4fbe-a9b3-43020b09e8c9","remote_addr":"[fd00:1122:3344:101::c]:42830","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":121,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:31.987723181Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:51540"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:31.987874535Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"503e33d8-fa8d-4aa5-9b09-ce5c78253a44","remote_addr":"[fd00:1122:3344:101::a]:51540","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":111,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:32.045809902Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:50690"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:32.045904695Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"534aec51-be5c-49df-8ca3-4766499e3565","remote_addr":"[fd00:1122:3344:101::a]:50690","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":67,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:32.053717518Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"b60d3345-ff52-462e-91a1-4e895359a411","remote_addr":"[fd00:1122:3344:101::a]:50690","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":112,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:32.228536156Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:41171"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:32.279162603Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"73b3b58d-3230-4dda-9e91-7d9bb2dcc438","remote_addr":"[fd00:1122:3344:101::c]:41171","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":50574,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:32.279286157Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:64078"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:32.327299596Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"5d820f71-7508-4819-9a6f-bdcf9bd857b2","remote_addr":"[fd00:1122:3344:101::a]:64078","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":47954,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:32.327601604Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"2b80ec5c-5536-49a2-802d-25aaf5f276cd","remote_addr":"[fd00:1122:3344:101::c]:41171","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":135,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:32.327746958Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"a22c966c-039e-4e2c-81ee-43e11f40e746","remote_addr":"[fd00:1122:3344:101::a]:64078","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":206,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:32.826308439Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:62200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:32.826507714Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"41cf05fb-f435-4263-9c2e-4391bbf13101","remote_addr":"[fd00:1122:3344:101::b]:62200","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":186,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:32.917904201Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:54476"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:32.917994574Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"dbe01a50-7210-4cac-aa88-8be45145774d","remote_addr":"[fd00:1122:3344:101::b]:54476","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":65,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:32.925991772Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"b2ead5d6-3fa3-481c-a71e-0d0e6cd75995","remote_addr":"[fd00:1122:3344:101::b]:54476","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":718,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:33.157625858Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:39458"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:33.204641202Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"cb3c89cc-6288-4ce4-a65f-f3f6ba9383f6","remote_addr":"[fd00:1122:3344:101::b]:39458","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":46979,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:33.204929819Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"64ba039c-9a95-4cbd-ae5c-48632cf6cb9d","remote_addr":"[fd00:1122:3344:101::b]:39458","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":101,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:44.592185492Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:46497"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:44.592316316Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"c7e406bb-e2f0-48d9-8ce9-d2fbcd9ee85c","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:46497","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":100,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:51:50.176913229Z","hostname":"buskin","pid":1555,"uri":"/0c1108fb-cc91-4535-adf7-846d50ef8716","method":"GET","req_id":"19883cfd-aec1-4b77-8984-64b45b516051","remote_addr":"[fd00:1122:3344:101::d]:57512","local_addr":"[fd00:1122:3344:101::1]:56984","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":1618,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:01.864786341Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:51469"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:01.864973896Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"e8b28e12-8b55-40a5-b500-05d69a62f1dc","remote_addr":"[fd00:1122:3344:101::a]:51469","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":158,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:01.866160687Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:49381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:01.866229129Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"0323c7e8-d77c-4ea7-be8c-8db38a4cff0d","remote_addr":"[fd00:1122:3344:101::a]:49381","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":46,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:01.874915965Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"92d50d4e-7c61-4900-a15b-e9062d177b09","remote_addr":"[fd00:1122:3344:101::a]:49381","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":713,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:01.964243109Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:57109"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:01.9642952Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"38907a50-8b2e-4219-9f9c-a58cc021c22a","remote_addr":"[fd00:1122:3344:101::c]:57109","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":32,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:01.972770471Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"71e78500-2a03-4f5a-8c8f-1f62fa0e16f3","remote_addr":"[fd00:1122:3344:101::c]:57109","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":737,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:01.984426364Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:36735"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:02.035043471Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"6bd8c47a-8f02-4fa6-ba0c-ade3167727cf","remote_addr":"[fd00:1122:3344:101::a]:36735","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":50581,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:02.035982185Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"d6cdf3ac-aa46-454d-b275-21115ef8fee6","remote_addr":"[fd00:1122:3344:101::a]:36735","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":680,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:02.04386286Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:39743"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:02.044091456Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"729671a9-42b9-422b-bde8-089797ca2e0a","remote_addr":"[fd00:1122:3344:101::c]:39743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":178,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:02.141574143Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:34396"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:02.187320363Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"befb97e3-f388-49a4-ab08-3c6e4eaf82a0","remote_addr":"[fd00:1122:3344:101::c]:34396","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":45689,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:02.187561979Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"49f919cd-6eb2-4847-9b40-e65f6b8d7bee","remote_addr":"[fd00:1122:3344:101::c]:34396","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":97,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:02.58696169Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:61071"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:02.587854143Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"888ebaab-54c2-4359-86d5-57195db9001c","remote_addr":"[fd00:1122:3344:101::b]:61071","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":855,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:02.596137479Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"22d42cc0-d62d-4294-a7a9-252e8d329e05","remote_addr":"[fd00:1122:3344:101::b]:61071","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":149,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:02.747337763Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:61807"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:02.747499387Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"d4ba6264-4fe5-472f-b6b1-808c2f27b351","remote_addr":"[fd00:1122:3344:101::b]:61807","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":125,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:02.853550226Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:57969"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:02.902062448Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"c52dab35-f83b-4878-bb03-f859933cc5e2","remote_addr":"[fd00:1122:3344:101::b]:57969","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":48449,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:02.902323765Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"29a46aea-18b7-4bde-add7-c8fcdf634119","remote_addr":"[fd00:1122:3344:101::b]:57969","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":102,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:14.626314837Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:42303"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:14.626967474Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"10fd0f6f-0fab-4b9d-b15a-a1dc7324fbb6","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:42303","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":628,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:20.175780618Z","hostname":"buskin","pid":1555,"uri":"/0c1108fb-cc91-4535-adf7-846d50ef8716","method":"GET","req_id":"abfc3581-7a57-4271-bf69-3c83d49fd81d","remote_addr":"[fd00:1122:3344:101::d]:57512","local_addr":"[fd00:1122:3344:101::1]:56984","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":963,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:32.038456667Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:60657"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:32.038602531Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"208a09d2-caff-4812-816b-32aea0a290d9","remote_addr":"[fd00:1122:3344:101::c]:60657","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":131,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:32.047510693Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"68d5a403-3726-4185-b32e-83ab01aa55c3","remote_addr":"[fd00:1122:3344:101::c]:60657","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":986,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:32.091138568Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:47276"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:32.091309212Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:673","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 005b2740-84d5-473e-ae59-aa79fd2fe5b6, kind: Service { id: 09fdc6fd-8297-436e-903d-b18710599203 }, name: Name(\"external-dns-09fdc6fd-8297-436e-903d-b18710599203\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 59731bd7-ffc9-493f-a33c-a1cf5ba2df30, kind: Service { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1 }, name: Name(\"external-dns-1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 32])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: c6edf68c-f558-4f51-9aed-6eb91fb3ad51, kind: Service { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52 }, name: Name(\"nexus-87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 117])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e6898038-bf11-41c8-b971-6ff3eed6d38b, kind: Service { id: 6088410e-a8fc-4d33-b593-6385e863291f }, name: Name(\"nexus-6088410e-a8fc-4d33-b593-6385e863291f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 27])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ffe6d41d-1349-42b2-a887-49d734eef422, kind: Service { id: 742e0330-77de-4723-a145-74751d41c73b }, name: Name(\"nexus-742e0330-77de-4723-a145-74751d41c73b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:32.091352344Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:32.091391775Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:32.091409865Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:32.091420165Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:32.091431916Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:32.091450636Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:32.091476887Z","hostname":"buskin","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"40a9f429-9ee8-4a9f-9a68-02eb09970e57","remote_addr":"[fd00:1122:3344:101::c]:47276","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":301,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:32.098205992Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:61272"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:32.098381027Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"f4df8d7d-4aaa-4951-b36b-cf88e65bd6ac","remote_addr":"[fd00:1122:3344:101::a]:61272","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":128,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:32.103427348Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:56119"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:32.104602578Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"e5831014-9b6f-428e-9b3b-ac7f20d0abe1","remote_addr":"[fd00:1122:3344:101::c]:56119","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":1145,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:32.135270166Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:57277"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:32.136156639Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:673","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 005b2740-84d5-473e-ae59-aa79fd2fe5b6, kind: Service { id: 09fdc6fd-8297-436e-903d-b18710599203 }, name: Name(\"external-dns-09fdc6fd-8297-436e-903d-b18710599203\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 59731bd7-ffc9-493f-a33c-a1cf5ba2df30, kind: Service { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1 }, name: Name(\"external-dns-1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 32])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: c6edf68c-f558-4f51-9aed-6eb91fb3ad51, kind: Service { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52 }, name: Name(\"nexus-87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 117])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e6898038-bf11-41c8-b971-6ff3eed6d38b, kind: Service { id: 6088410e-a8fc-4d33-b593-6385e863291f }, name: Name(\"nexus-6088410e-a8fc-4d33-b593-6385e863291f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 27])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ffe6d41d-1349-42b2-a887-49d734eef422, kind: Service { id: 742e0330-77de-4723-a145-74751d41c73b }, name: Name(\"nexus-742e0330-77de-4723-a145-74751d41c73b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:32.13618334Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:32.136218841Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:32.136228601Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:32.136241192Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:32.136259182Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:32.136270022Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:32.136285443Z","hostname":"buskin","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"3e167fdb-1ae8-43d8-aa3c-e01a4ed7df43","remote_addr":"[fd00:1122:3344:101::a]:57277","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":960,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:32.186655183Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:34719"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:32.186718305Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"864e75b2-8eb4-48a9-abe9-83faa06e9cee","remote_addr":"[fd00:1122:3344:101::a]:34719","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":42,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:32.194449646Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"5a95188a-bc16-40ff-aec7-5cf1723aa7a9","remote_addr":"[fd00:1122:3344:101::a]:34719","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":84,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:32.34452612Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:33115"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:32.39521969Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"d7f2ec1b-f504-43c9-9977-9c38a8fe5a50","remote_addr":"[fd00:1122:3344:101::a]:33115","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":50649,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:32.395340713Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:43770"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:32.399335546Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"ba8defe1-9c2d-42f8-9ff7-ab278e449bc0","remote_addr":"[fd00:1122:3344:101::a]:33115","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":3756,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:32.441208776Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"453ec7a2-20d6-48b4-819f-e4ff1e5af78b","remote_addr":"[fd00:1122:3344:101::c]:43770","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":45775,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:32.441777761Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"5b53ac9d-7689-45bf-a8c6-cdc1d3139f2b","remote_addr":"[fd00:1122:3344:101::c]:43770","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":285,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:32.757514095Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:50858"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:32.757716931Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"192b9f44-4845-44d1-93ad-117d590fd462","remote_addr":"[fd00:1122:3344:101::b]:50858","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":180,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:32.879111989Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:47177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:32.880094194Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"b01e5b82-dfab-49ad-b608-f8e917129b03","remote_addr":"[fd00:1122:3344:101::b]:47177","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":928,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:32.887197579Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"b625311c-4e95-4534-9e9f-57fe02112a75","remote_addr":"[fd00:1122:3344:101::b]:47177","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":130,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:32.954646554Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:54576"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:32.954818458Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:673","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 005b2740-84d5-473e-ae59-aa79fd2fe5b6, kind: Service { id: 09fdc6fd-8297-436e-903d-b18710599203 }, name: Name(\"external-dns-09fdc6fd-8297-436e-903d-b18710599203\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 187, 6])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 59731bd7-ffc9-493f-a33c-a1cf5ba2df30, kind: Service { id: 1eb90f9c-ba99-428e-885b-5fa2df7ae2f1 }, name: Name(\"external-dns-1eb90f9c-ba99-428e-885b-5fa2df7ae2f1\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 175, 32])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: c6edf68c-f558-4f51-9aed-6eb91fb3ad51, kind: Service { id: 87d65a87-e1e5-43e4-9cba-b4afd5e9ac52 }, name: Name(\"nexus-87d65a87-e1e5-43e4-9cba-b4afd5e9ac52\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 163, 117])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e6898038-bf11-41c8-b971-6ff3eed6d38b, kind: Service { id: 6088410e-a8fc-4d33-b593-6385e863291f }, name: Name(\"nexus-6088410e-a8fc-4d33-b593-6385e863291f\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 171, 27])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ffe6d41d-1349-42b2-a887-49d734eef422, kind: Service { id: 742e0330-77de-4723-a145-74751d41c73b }, name: Name(\"nexus-742e0330-77de-4723-a145-74751d41c73b\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 184])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:32.9548675Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:32.954906231Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:32.954914401Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:32.954932132Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:32.954957682Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:32.954979113Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:32.954997193Z","hostname":"buskin","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"8d130d79-1d28-4339-9bf5-7335b9a00686","remote_addr":"[fd00:1122:3344:101::b]:54576","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":329,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:33.225175992Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:62765"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:33.271907318Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"91fd31e5-4c57-417f-a948-d95a37ce08e2","remote_addr":"[fd00:1122:3344:101::b]:62765","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":46679,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:33.272822792Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"e5440c54-f664-458f-89e2-7fa76f4426e4","remote_addr":"[fd00:1122:3344:101::b]:62765","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":772,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:44.658998305Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:48523"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:44.659121279Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"1e860fb5-a4d7-4d77-8c83-358b7f93548d","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:48523","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":95,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:52:50.175514619Z","hostname":"buskin","pid":1555,"uri":"/0c1108fb-cc91-4535-adf7-846d50ef8716","method":"GET","req_id":"cc426885-3f2d-44b1-b64a-bd21f141385a","remote_addr":"[fd00:1122:3344:101::d]:57512","local_addr":"[fd00:1122:3344:101::1]:56984","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":915,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:01.885915465Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:56204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:01.886096689Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"f53a60ae-e3a2-465e-983e-27ab07ba9708","remote_addr":"[fd00:1122:3344:101::a]:56204","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":168,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:01.930481714Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:53645"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:01.930571496Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"131fabda-0b93-4975-ad34-6d3b98a7313a","remote_addr":"[fd00:1122:3344:101::c]:53645","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":55,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:01.937396394Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"0808b8b2-6ddc-4c9f-b887-ff699be5b2f4","remote_addr":"[fd00:1122:3344:101::c]:53645","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":105,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:01.939685364Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:53092"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:01.939802476Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"0f2aef79-569f-44b2-86bb-16136d406cbe","remote_addr":"[fd00:1122:3344:101::c]:53092","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":82,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:01.968827002Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:55611"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:01.969713155Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"b4835a90-ea9a-49ea-9bcd-02447c718f72","remote_addr":"[fd00:1122:3344:101::a]:55611","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":840,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:01.976895332Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"0b5317a0-8b71-4da1-a449-5c9b16506811","remote_addr":"[fd00:1122:3344:101::a]:55611","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":65,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:02.012138998Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:44199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:02.061563055Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"157f14e7-5e77-4973-8eea-30b6e8340aa2","remote_addr":"[fd00:1122:3344:101::c]:44199","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":49380,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:02.061873612Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"110f9d37-b23e-4ddd-a3ec-8dc12a800ad8","remote_addr":"[fd00:1122:3344:101::c]:44199","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":121,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:02.0725335Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:43628"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:02.119750748Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"a1305584-139e-4b6c-a26b-14c90291619b","remote_addr":"[fd00:1122:3344:101::a]:43628","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":47101,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:02.120164129Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"4d681783-8830-461e-95a7-4f1c0ff25837","remote_addr":"[fd00:1122:3344:101::a]:43628","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":172,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:02.620156267Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:38240"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:02.620423794Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"cee052a6-0074-4fdb-9175-67116f15003c","remote_addr":"[fd00:1122:3344:101::b]:38240","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":232,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:02.694137532Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:40934"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:02.694249985Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"3908a8c3-6556-4163-ad28-dcafebf166df","remote_addr":"[fd00:1122:3344:101::b]:40934","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":84,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:02.701012561Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"370cda16-e5c3-4996-a2f8-1bf17899ac30","remote_addr":"[fd00:1122:3344:101::b]:40934","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":133,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:02.796993348Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:46555"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:02.846189288Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"55ef6062-26e6-4ce6-a4b2-4597e1be917e","remote_addr":"[fd00:1122:3344:101::b]:46555","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":49144,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:02.846447625Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"6e281a29-a018-4274-81cd-6189ede92ec1","remote_addr":"[fd00:1122:3344:101::b]:46555","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":102,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:14.690890308Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:65241"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:14.691113674Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"0464c8c0-7e7c-4431-9b04-0e008dee7b42","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:65241","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":56,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:20.175219731Z","hostname":"buskin","pid":1555,"uri":"/0c1108fb-cc91-4535-adf7-846d50ef8716","method":"GET","req_id":"4bc8361a-8d8b-4a0e-9581-dabefb0762d4","remote_addr":"[fd00:1122:3344:101::d]:57512","local_addr":"[fd00:1122:3344:101::1]:56984","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":606,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:31.920063344Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:57140"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:31.920182427Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"07d201c5-ccce-44c2-804d-40b950364e49","remote_addr":"[fd00:1122:3344:101::c]:57140","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":110,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:31.928185115Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"25b78066-b09c-4e6b-80db-75a22cb918bc","remote_addr":"[fd00:1122:3344:101::c]:57140","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":99,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:31.943246907Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:33735"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:31.943417291Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"d6db9058-d08e-4715-ba27-f11970257efa","remote_addr":"[fd00:1122:3344:101::a]:33735","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":122,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:32.066500634Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:50404"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:32.066589916Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"b4c75f6a-e1e8-435a-8333-719026f32b43","remote_addr":"[fd00:1122:3344:101::a]:50404","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":62,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:32.074184044Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"6b6f20cb-5495-458a-96ed-63c5a031a7b4","remote_addr":"[fd00:1122:3344:101::a]:50404","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":93,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:32.123307552Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:65123"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:32.123474146Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"d73bcfc3-4510-48b2-ac1f-5fe18789f410","remote_addr":"[fd00:1122:3344:101::c]:65123","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":114,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:32.256170678Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:49138"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:32.304570998Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"acd0460a-8064-4a56-93fa-f722fbb3b3b5","remote_addr":"[fd00:1122:3344:101::a]:49138","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":48352,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:32.304821094Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"43942805-d24d-49d7-b4d0-43cfc1035b61","remote_addr":"[fd00:1122:3344:101::a]:49138","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":100,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:32.38155273Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:41795"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:32.42805098Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"8acb2ce5-0961-41cb-95c4-c6741d1ce119","remote_addr":"[fd00:1122:3344:101::c]:41795","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":46403,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:32.428875922Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"553e4877-2121-4fce-aa33-7bf8cf02e9ae","remote_addr":"[fd00:1122:3344:101::c]:41795","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":681,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:32.743439266Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:43265"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:32.743738253Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"e970ceb1-e14e-4cb8-af27-194eedf18903","remote_addr":"[fd00:1122:3344:101::b]:43265","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":241,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:32.805218393Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:47804"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:32.806089416Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"dda3507d-a201-4ea9-a105-aeadf45a9592","remote_addr":"[fd00:1122:3344:101::b]:47804","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":831,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:32.813058047Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"4e5ae20a-e3a5-41ad-a366-723bb1e99a99","remote_addr":"[fd00:1122:3344:101::b]:47804","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":112,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:33.089719805Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:58224"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:33.134832518Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"1e094d24-1581-4553-867b-55d2ec1ba107","remote_addr":"[fd00:1122:3344:101::b]:58224","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":45038,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:33.135076635Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"1c74bf3d-2527-4719-b0c6-95ec1150a9ff","remote_addr":"[fd00:1122:3344:101::b]:58224","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":98,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:44.723896447Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:41748"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:44.72400958Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"e934dcb6-b677-484d-9285-49bd5cb87165","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:41748","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":88,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:50.175299865Z","hostname":"buskin","pid":1555,"uri":"/0c1108fb-cc91-4535-adf7-846d50ef8716","method":"GET","req_id":"dcfad900-f250-4c2c-863c-c4f5ca050675","remote_addr":"[fd00:1122:3344:101::d]:57512","local_addr":"[fd00:1122:3344:101::1]:56984","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"0c1108fb-cc91-4535-adf7-846d50ef8716","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":590,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:51.26363739Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:45941"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:51.263958299Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"08a3b64b-1dde-44a9-b0ef-8371b66566ab","remote_addr":"[fd00:1122:3344:101::a]:45941","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":270,"response_code":"200"} {"msg":"Mapping virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:51.264235086Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:780","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 248, 48, 19])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(11145136) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:51.264284857Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"PUT","req_id":"dca6206a-84d5-4471-8715-9612e1f710bb","remote_addr":"[fd00:1122:3344:101::a]:45941","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":137,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:51.389200387Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:55735"} {"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:51.408300274Z","hostname":"buskin","pid":1555,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:577","metadata":"InstanceMetadata { silo_id: feca8d2d-3fe6-4796-88fc-7959b0c126c0, project_id: 63fa7974-6cf8-470b-93fd-0e60179cdd83 }","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-07-19T17:53:50.962217Z }","instance_runtime":"InstanceRuntimeState { propolis_id: Some(df1fc3a9-614e-45dd-b799-a14257326be0 (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-07-19T17:53:51.021735627Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: c14dc971-a97e-42ef-9aa1-1e3ad1cec2d6, kind: Instance { id: ccff4298-547e-428a-b327-016613e4386e }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 248, 48, 19])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(11145136), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: c14dc971-a97e-42ef-9aa1-1e3ad1cec2d6, kind: Instance { id: ccff4298-547e-428a-b327-016613e4386e }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 248, 48, 19])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(11145136), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: c14dc971-a97e-42ef-9aa1-1e3ad1cec2d6, kind: Instance { id: ccff4298-547e-428a-b327-016613e4386e }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 248, 48, 19])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(11145136), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(11145136))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: c14dc971-a97e-42ef-9aa1-1e3ad1cec2d6, kind: Instance { id: ccff4298-547e-428a-b327-016613e4386e }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 248, 48, 19])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(11145136), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-2a5942cc1c6\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: f09b670c-44c1-4800-8f7b-23ddfd8beb2e, read_only_parent: Some(Volume { block_size: 512, id: 493cd121-797a-4616-b69a-81ed09b6179e, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 3dcbfd03-a6f5-476e-ba57-1bc968f1fb16, key: Some(\"ygWrIc4lwZIpVeVyvMB0gYlWiOsDO4JXA2srWgfKhI4=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::17]:19001\", \"[fd00:1122:3344:101::14]:19001\", \"[fd00:1122:3344:101::13]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: f09b670c-44c1-4800-8f7b-23ddfd8beb2e, key: Some(\"AwWPqJfHsTuoVrRgUuzUUppRfaElgQAMSTZup36zums=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::17]:19002\", \"[fd00:1122:3344:101::16]:19001\", \"[fd00:1122:3344:101::15]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAACHuY7zWPNYAAC5jvNYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAh7mO81jzWAAAuY7zWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiJjY2ZmNDI5OC01NDdlLTQyOGEtYjMyNy0wMTY2MTNlNDM4NmUiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJSXhxak52cmtDSlNsanNSQ0luK21FNm50TzA3eUpybzM3TXdzQ2VVWXB6OSJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }","propolis_id":"df1fc3a9-614e-45dd-b799-a14257326be0","instance_id":"ccff4298-547e-428a-b327-016613e4386e"} {"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:51.408396566Z","hostname":"buskin","pid":1555,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:611","instance_id":"ccff4298-547e-428a-b327-016613e4386e (instance)"} {"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:51.408669204Z","hostname":"buskin","pid":1555,"component":"InstanceManager","file":"sled-agent/src/instance.rs:951","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: c14dc971-a97e-42ef-9aa1-1e3ad1cec2d6, kind: Instance { id: ccff4298-547e-428a-b327-016613e4386e }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 248, 48, 19])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(11145136), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: c14dc971-a97e-42ef-9aa1-1e3ad1cec2d6, kind: Instance { id: ccff4298-547e-428a-b327-016613e4386e }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 248, 48, 19])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(11145136), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: c14dc971-a97e-42ef-9aa1-1e3ad1cec2d6, kind: Instance { id: ccff4298-547e-428a-b327-016613e4386e }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 248, 48, 19])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(11145136), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(11145136))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: c14dc971-a97e-42ef-9aa1-1e3ad1cec2d6, kind: Instance { id: ccff4298-547e-428a-b327-016613e4386e }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 248, 48, 19])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(11145136), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-2a5942cc1c6\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: f09b670c-44c1-4800-8f7b-23ddfd8beb2e, read_only_parent: Some(Volume { block_size: 512, id: 493cd121-797a-4616-b69a-81ed09b6179e, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 3dcbfd03-a6f5-476e-ba57-1bc968f1fb16, key: Some(\"ygWrIc4lwZIpVeVyvMB0gYlWiOsDO4JXA2srWgfKhI4=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::17]:19001\", \"[fd00:1122:3344:101::14]:19001\", \"[fd00:1122:3344:101::13]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: f09b670c-44c1-4800-8f7b-23ddfd8beb2e, key: Some(\"AwWPqJfHsTuoVrRgUuzUUppRfaElgQAMSTZup36zums=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::17]:19002\", \"[fd00:1122:3344:101::16]:19001\", \"[fd00:1122:3344:101::15]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAACHuY7zWPNYAAC5jvNYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAh7mO81jzWAAAuY7zWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiJjY2ZmNDI5OC01NDdlLTQyOGEtYjMyNy0wMTY2MTNlNDM4NmUiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJSXhxak52cmtDSlNsanNSQ0luK21FNm50TzA3eUpybzM3TXdzQ2VVWXB6OSJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }, instance_runtime: InstanceRuntimeState { propolis_id: Some(df1fc3a9-614e-45dd-b799-a14257326be0 (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-07-19T17:53:51.021735627Z }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-07-19T17:53:50.962217Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400 }","propolis_id":"df1fc3a9-614e-45dd-b799-a14257326be0","instance_id":"ccff4298-547e-428a-b327-016613e4386e"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:51.410478691Z","hostname":"buskin","pid":1555,"uri":"/instances/ccff4298-547e-428a-b327-016613e4386e","method":"PUT","req_id":"89119799-e295-4f7d-81b4-55abcd814afc","remote_addr":"[fd00:1122:3344:101::a]:55735","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":21182,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:51.521866929Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:57914"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:51.523519741Z","hostname":"buskin","pid":1555,"uri":"/instances/ccff4298-547e-428a-b327-016613e4386e/state","method":"PUT","req_id":"4e4243e1-f550-4fbc-a7a7-097d37d76596","remote_addr":"[fd00:1122:3344:101::a]:57914","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":1577,"response_code":"200"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:51.52921623Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:426","port":"Port { inner: PortInner(PortData { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 248, 48, 19]), slot: 0, vni: Vni { inner: 11145136 }, subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 } }) }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:51.529851486Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Configuring new Omicron zone: oxz_propolis-server_df1fc3a9-614e-45dd-b799-a14257326be0","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:51.567087585Z","hostname":"buskin","pid":1555,"instance_id":"ccff4298-547e-428a-b327-016613e4386e","component":"InstanceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_propolis-server_df1fc3a9-614e-45dd-b799-a14257326be0","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:51.59111369Z","hostname":"buskin","pid":1555,"instance_id":"ccff4298-547e-428a-b327-016613e4386e","component":"InstanceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_propolis-server_df1fc3a9-614e-45dd-b799-a14257326be0:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:52.901481612Z","hostname":"buskin","pid":1555,"instance_id":"ccff4298-547e-428a-b327-016613e4386e","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:52.902365155Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_df1fc3a9-614e-45dd-b799-a14257326be0","instance_id":"ccff4298-547e-428a-b327-016613e4386e","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"wait for service Some(\"oxz_propolis-server_df1fc3a9-614e-45dd-b799-a14257326be0\") failed: Property not found. retry in 40.739378ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:53:53.156752103Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_df1fc3a9-614e-45dd-b799-a14257326be0","instance_id":"ccff4298-547e-428a-b327-016613e4386e","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_df1fc3a9-614e-45dd-b799-a14257326be0\") failed: Property not found. retry in 113.967665ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:53:53.209460034Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_df1fc3a9-614e-45dd-b799-a14257326be0","instance_id":"ccff4298-547e-428a-b327-016613e4386e","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_df1fc3a9-614e-45dd-b799-a14257326be0\") failed: Property not found. retry in 257.074395ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:53:53.337992028Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_df1fc3a9-614e-45dd-b799-a14257326be0","instance_id":"ccff4298-547e-428a-b327-016613e4386e","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_df1fc3a9-614e-45dd-b799-a14257326be0\") failed: Property not found. retry in 407.8322ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:53:53.606622927Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_df1fc3a9-614e-45dd-b799-a14257326be0","instance_id":"ccff4298-547e-428a-b327-016613e4386e","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_df1fc3a9-614e-45dd-b799-a14257326be0\") failed: Property not found. retry in 1.002608194s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:53:54.024915959Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_df1fc3a9-614e-45dd-b799-a14257326be0","instance_id":"ccff4298-547e-428a-b327-016613e4386e","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_df1fc3a9-614e-45dd-b799-a14257326be0\") failed: Property not found. retry in 1.009936496s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:53:55.03889559Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_df1fc3a9-614e-45dd-b799-a14257326be0","instance_id":"ccff4298-547e-428a-b327-016613e4386e","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_df1fc3a9-614e-45dd-b799-a14257326be0\") failed: Property not found. retry in 1.422586579s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:53:56.058919857Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_df1fc3a9-614e-45dd-b799-a14257326be0","instance_id":"ccff4298-547e-428a-b327-016613e4386e","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_df1fc3a9-614e-45dd-b799-a14257326be0\") failed: Property not found. retry in 1.127368572s","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:53:57.490708367Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_df1fc3a9-614e-45dd-b799-a14257326be0","instance_id":"ccff4298-547e-428a-b327-016613e4386e","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Started propolis in zone: oxz_propolis-server_df1fc3a9-614e-45dd-b799-a14257326be0","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:58.645151272Z","hostname":"buskin","pid":1555,"instance_id":"ccff4298-547e-428a-b327-016613e4386e","component":"InstanceManager","file":"sled-agent/src/instance.rs:1460"} {"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:58.653178861Z","hostname":"buskin","pid":1555,"instance_id":"ccff4298-547e-428a-b327-016613e4386e","component":"InstanceManager","file":"sled-agent/src/instance.rs:1469"} {"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:58.688422108Z","hostname":"buskin","pid":1555,"instance_id":"ccff4298-547e-428a-b327-016613e4386e","component":"InstanceManager","file":"sled-agent/src/instance.rs:1483"} {"msg":"Sending ensure request to propolis: InstanceEnsureRequest { cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAACHuY7zWPNYAAC5jvNYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAh7mO81jzWAAAuY7zWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiJjY2ZmNDI5OC01NDdlLTQyOGEtYjMyNy0wMTY2MTNlNDM4NmUiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJSXhxak52cmtDSlNsanNSQ0luK21FNm50TzA3eUpybzM3TXdzQ2VVWXB6OSJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"), disks: [DiskRequest { device: \"nvme\", name: \"disk-2a5942cc1c6\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: f09b670c-44c1-4800-8f7b-23ddfd8beb2e, read_only_parent: Some(Volume { block_size: 512, id: 493cd121-797a-4616-b69a-81ed09b6179e, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 3dcbfd03-a6f5-476e-ba57-1bc968f1fb16, key: Some(\"ygWrIc4lwZIpVeVyvMB0gYlWiOsDO4JXA2srWgfKhI4=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::17]:19001\", \"[fd00:1122:3344:101::14]:19001\", \"[fd00:1122:3344:101::13]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: f09b670c-44c1-4800-8f7b-23ddfd8beb2e, key: Some(\"AwWPqJfHsTuoVrRgUuzUUppRfaElgQAMSTZup36zums=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::17]:19002\", \"[fd00:1122:3344:101::16]:19001\", \"[fd00:1122:3344:101::15]:19000\"] } }] } }], migrate: None, nics: [NetworkInterfaceRequest { name: \"opte6\", slot: Slot(0) }], properties: InstanceProperties { bootrom_id: 00000000-0000-0000-0000-000000000000, description: \"Test description\", id: ccff4298-547e-428a-b327-016613e4386e, image_id: 00000000-0000-0000-0000-000000000000, memory: 1024, metadata: InstanceMetadata { project_id: 63fa7974-6cf8-470b-93fd-0e60179cdd83, silo_id: feca8d2d-3fe6-4796-88fc-7959b0c126c0 }, name: \"localshark\", vcpus: 2 } }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:58.693915121Z","hostname":"buskin","pid":1555,"instance_id":"ccff4298-547e-428a-b327-016613e4386e","component":"InstanceManager","file":"sled-agent/src/instance.rs:698"} {"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:58.866408209Z","hostname":"buskin","pid":1555,"instance_id":"ccff4298-547e-428a-b327-016613e4386e","component":"InstanceManager","file":"sled-agent/src/instance.rs:700"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Starting), migration_status: NoMigration, time: 2024-07-19T17:53:58.867830726Z }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:58.867871417Z","hostname":"buskin","pid":1555,"instance_id":"ccff4298-547e-428a-b327-016613e4386e","component":"InstanceManager","file":"sled-agent/src/instance.rs:584"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:58.867883127Z","hostname":"buskin","pid":1555,"instance_id":"ccff4298-547e-428a-b327-016613e4386e","component":"InstanceManager","file":"sled-agent/src/instance.rs:606","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-07-19T17:53:58.867830726Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(df1fc3a9-614e-45dd-b799-a14257326be0 (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-07-19T17:53:51.021735627Z }","propolis_id":"df1fc3a9-614e-45dd-b799-a14257326be0"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:58.867893957Z","hostname":"buskin","pid":1555,"instance_id":"ccff4298-547e-428a-b327-016613e4386e","component":"InstanceManager","file":"sled-agent/src/instance.rs:503","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(df1fc3a9-614e-45dd-b799-a14257326be0 (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-07-19T17:53:51.021735627Z }, propolis_id: df1fc3a9-614e-45dd-b799-a14257326be0 (propolis), vmm_state: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-07-19T17:53:58.867830726Z }, migration_state: None }","instance_id":"ccff4298-547e-428a-b327-016613e4386e"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_status: NoMigration, time: 2024-07-19T17:53:58.965184639Z }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:58.965198669Z","hostname":"buskin","pid":1555,"instance_id":"ccff4298-547e-428a-b327-016613e4386e","component":"InstanceManager","file":"sled-agent/src/instance.rs:584"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:58.965218389Z","hostname":"buskin","pid":1555,"instance_id":"ccff4298-547e-428a-b327-016613e4386e","component":"InstanceManager","file":"sled-agent/src/instance.rs:606","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-07-19T17:53:58.965184639Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(df1fc3a9-614e-45dd-b799-a14257326be0 (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-07-19T17:53:51.021735627Z }","propolis_id":"df1fc3a9-614e-45dd-b799-a14257326be0"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:58.96524118Z","hostname":"buskin","pid":1555,"instance_id":"ccff4298-547e-428a-b327-016613e4386e","component":"InstanceManager","file":"sled-agent/src/instance.rs:503","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(df1fc3a9-614e-45dd-b799-a14257326be0 (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-07-19T17:53:51.021735627Z }, propolis_id: df1fc3a9-614e-45dd-b799-a14257326be0 (propolis), vmm_state: VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-07-19T17:53:58.965184639Z }, migration_state: None }","instance_id":"ccff4298-547e-428a-b327-016613e4386e"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:59.001873613Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:62107"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:59.001978326Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"124151e2-3290-4afa-ad5a-def894c85e9a","remote_addr":"[fd00:1122:3344:101::c]:62107","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":55,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:59.048979279Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"3d7b9138-9e54-46a9-a755-8d3640f2e361","remote_addr":"[fd00:1122:3344:101::c]:62107","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":827,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:59.085206141Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:50310"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:59.085314784Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"0f0ed64e-56b3-4824-b89e-b8593b731f99","remote_addr":"[fd00:1122:3344:101::c]:50310","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":47,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:53:59.097234474Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"af32d44d-d878-4550-816a-1141663c7b90","remote_addr":"[fd00:1122:3344:101::c]:50310","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":69,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:01.864527279Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:53214"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:01.864653873Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"013ba19a-52a3-408a-af4f-b01adf595d3c","remote_addr":"[fd00:1122:3344:101::c]:53214","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":116,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:01.879420017Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"4381620b-4046-4bc3-a337-9cedda3a339e","remote_addr":"[fd00:1122:3344:101::c]:53214","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":132,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:01.884578551Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:65101"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:01.884738755Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"c3939abd-5916-4a18-87c4-f1e24b8d1d4b","remote_addr":"[fd00:1122:3344:101::c]:65101","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":120,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:01.893082082Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:37907"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:01.894326465Z","hostname":"buskin","pid":1555,"uri":"/instances/ccff4298-547e-428a-b327-016613e4386e/state","method":"GET","req_id":"eb99d0c4-f75a-4430-8698-9464467f25af","remote_addr":"[fd00:1122:3344:101::c]:37907","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":1189,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:01.896216014Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:47507"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:01.89646309Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"9d62c797-ae6f-4b4e-b574-9d7108c99dd5","remote_addr":"[fd00:1122:3344:101::a]:47507","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":190,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:02.090801957Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:43503"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:02.141539177Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"2c4597cd-3224-48ee-9848-984203002953","remote_addr":"[fd00:1122:3344:101::c]:43503","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":50667,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:02.141820334Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"64138975-7e35-472a-9efd-1a0c40e1378d","remote_addr":"[fd00:1122:3344:101::c]:43503","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":97,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:02.182747399Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:44697"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:02.182968395Z","hostname":"buskin","pid":1555,"uri":"/instances/ccff4298-547e-428a-b327-016613e4386e/state","method":"GET","req_id":"bfc10dab-7f39-4b3d-807a-b82cd39b2fcf","remote_addr":"[fd00:1122:3344:101::a]:44697","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":175,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:02.186295991Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:49885"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:02.187153683Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"f8ff2b36-466b-4058-a38c-3d766c42016e","remote_addr":"[fd00:1122:3344:101::a]:49885","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":792,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:02.202112153Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"79a42162-e45f-4b3e-b9a4-7463261eaeb3","remote_addr":"[fd00:1122:3344:101::a]:49885","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":129,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:02.307150685Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:42826"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:02.356838028Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"88f8b306-b3ba-4f2a-b7dc-241418d097ff","remote_addr":"[fd00:1122:3344:101::a]:42826","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":49640,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:02.357144756Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"17f6907f-3537-4a99-85d0-7d2ceb12ee01","remote_addr":"[fd00:1122:3344:101::a]:42826","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":120,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:02.831480227Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:59488"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:02.831682852Z","hostname":"buskin","pid":1555,"uri":"/instances/ccff4298-547e-428a-b327-016613e4386e/state","method":"GET","req_id":"fed2c98c-d2f5-477c-98b0-28ebb44890f7","remote_addr":"[fd00:1122:3344:101::b]:59488","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":185,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:02.848060068Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:47109"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:02.848188751Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"a00f11ac-8bcf-476b-8c22-ee4ba04ce3dc","remote_addr":"[fd00:1122:3344:101::b]:47109","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":80,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:02.850547392Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:58492"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:02.850741317Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"59d9dd35-c551-4930-b50a-edd8dc4539bc","remote_addr":"[fd00:1122:3344:101::b]:58492","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":144,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:02.861834516Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"fb11cacf-251c-45f8-aea5-b8691fabd1f3","remote_addr":"[fd00:1122:3344:101::b]:47109","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":740,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:02.903651013Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:34841"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:02.952146115Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"b6b01abc-73d8-4fd3-9f3f-13ef7a6a4b4a","remote_addr":"[fd00:1122:3344:101::b]:34841","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":48447,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:02.953146481Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"9b281940-17da-4ac0-99a6-3449634c3d8b","remote_addr":"[fd00:1122:3344:101::b]:34841","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":815,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:09.102159697Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:39415"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:09.102736172Z","hostname":"buskin","pid":1555,"uri":"/instances/ccff4298-547e-428a-b327-016613e4386e/state","method":"PUT","req_id":"366ccba2-179e-4f9f-9dc8-6374b66a40ac","remote_addr":"[fd00:1122:3344:101::a]:39415","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":547,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_status: NoMigration, time: 2024-07-19T17:54:09.102834165Z }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:09.102892517Z","hostname":"buskin","pid":1555,"instance_id":"ccff4298-547e-428a-b327-016613e4386e","component":"InstanceManager","file":"sled-agent/src/instance.rs:584"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:09.102928497Z","hostname":"buskin","pid":1555,"instance_id":"ccff4298-547e-428a-b327-016613e4386e","component":"InstanceManager","file":"sled-agent/src/instance.rs:606","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-07-19T17:54:09.102834165Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(df1fc3a9-614e-45dd-b799-a14257326be0 (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-07-19T17:53:51.021735627Z }","propolis_id":"df1fc3a9-614e-45dd-b799-a14257326be0"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:09.102944638Z","hostname":"buskin","pid":1555,"instance_id":"ccff4298-547e-428a-b327-016613e4386e","component":"InstanceManager","file":"sled-agent/src/instance.rs:503","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(df1fc3a9-614e-45dd-b799-a14257326be0 (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-07-19T17:53:51.021735627Z }, propolis_id: df1fc3a9-614e-45dd-b799-a14257326be0 (propolis), vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-07-19T17:54:09.102834165Z }, migration_state: None }","instance_id":"ccff4298-547e-428a-b327-016613e4386e"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_status: NoMigration, time: 2024-07-19T17:54:09.156760378Z }","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:09.156787089Z","hostname":"buskin","pid":1555,"instance_id":"ccff4298-547e-428a-b327-016613e4386e","component":"InstanceManager","file":"sled-agent/src/instance.rs:584"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:09.15681815Z","hostname":"buskin","pid":1555,"instance_id":"ccff4298-547e-428a-b327-016613e4386e","component":"InstanceManager","file":"sled-agent/src/instance.rs:606","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(6), time_updated: 2024-07-19T17:54:09.156760378Z }","new_instance_state":"InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-07-19T17:54:09.156760378Z }","propolis_id":"df1fc3a9-614e-45dd-b799-a14257326be0"} {"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:09.15684209Z","hostname":"buskin","pid":1555,"instance_id":"ccff4298-547e-428a-b327-016613e4386e","component":"InstanceManager","file":"sled-agent/src/instance.rs:621","instance_id":"ccff4298-547e-428a-b327-016613e4386e"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:09.157528698Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:09.160444744Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:09.161382348Z","hostname":"buskin","pid":1555,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:445","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/20dccfdd-e68a-468d-8629-7f7e0834aa1c/debug/bundle/zone\", \"/pool/int/301424fa-c483-4ce9-92fb-3c8ee10e6f9f/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/988e775a-ddc4-455d-bc4e-f2e3c228cc74/crypt/debug\", \"/pool/ext/f3be927c-0009-45ce-8f85-0c94f9b67b8f/crypt/debug\", \"/pool/ext/ef4ff4d0-0a2e-4db9-b6d2-7a22e69d7765/crypt/debug\", \"/pool/ext/4c50992d-c53d-42b7-a46f-5340e3aee556/crypt/debug\", \"/pool/ext/7c02949e-12b1-4d82-9ac3-2efd807aef95/crypt/debug\", \"/pool/ext/765bc25e-72d0-4d30-8cfe-a58b7db5c2e8/crypt/debug\", \"/pool/ext/696d009b-6d0c-4082-b81b-231c6c85d568/crypt/debug\", \"/pool/ext/311acf6c-db4a-40cb-bd1f-e33d2bc8747f/crypt/debug\", \"/pool/ext/41faf794-cdf9-49a0-b529-4c18e335c28c/crypt/debug\"] }","zone_name":"oxz_propolis-server_df1fc3a9-614e-45dd-b799-a14257326be0"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:09.195442364Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:35741"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:09.195708922Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"6aadd0ad-53cf-4510-ba43-e46435c4b808","remote_addr":"[fd00:1122:3344:101::c]:35741","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":173,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:09.209770897Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"343dd6d0-79e2-4c8b-931b-86c9b018403b","remote_addr":"[fd00:1122:3344:101::c]:35741","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":161,"response_code":"204"} {"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:09.806613335Z","hostname":"buskin","pid":1555,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1246","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_df1fc3a9-614e-45dd-b799-a14257326be0\", bundle_id: 6303eb3d-c4fd-4b0f-9f33-bede4fa5ca59 }, time_created: 2024-07-19T17:54:09.161753248Z, version: 0, cause: TerminatedInstance }"} {"msg":"Halting and removing zone: oxz_propolis-server_df1fc3a9-614e-45dd-b799-a14257326be0","v":0,"name":"SledAgent","level":40,"time":"2024-07-19T17:54:09.806703238Z","hostname":"buskin","pid":1555,"instance_id":"ccff4298-547e-428a-b327-016613e4386e","component":"InstanceManager","file":"sled-agent/src/instance.rs:793"} {"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:11.536503782Z","hostname":"buskin","pid":1555,"instance_id":"ccff4298-547e-428a-b327-016613e4386e","component":"InstanceManager","file":"illumos-utils/src/zone.rs:270"} {"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:11.55218134Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:906","id":"RouterId { vni: Vni(11145136), kind: System }"} {"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:11.552213071Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:906","id":"RouterId { vni: Vni(11145136), kind: Custom(V4(Ipv4Net { addr: 172.30.0.0, width: 22 })) }"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:11.552715404Z","hostname":"buskin","pid":1555,"instance_id":"ccff4298-547e-428a-b327-016613e4386e","component":"InstanceManager","file":"sled-agent/src/instance.rs:503","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-07-19T17:54:09.156760378Z }, propolis_id: df1fc3a9-614e-45dd-b799-a14257326be0 (propolis), vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-07-19T17:54:11.552693243Z }, migration_state: None }","instance_id":"ccff4298-547e-428a-b327-016613e4386e"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:11.654553923Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:52746"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:11.6548151Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"fa6d3f81-e7c2-40a2-8e04-0cf74fe6aabe","remote_addr":"[fd00:1122:3344:101::c]:52746","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":209,"response_code":"200"} {"msg":"Mapping virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:11.655524088Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:780","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 248, 48, 19])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(11145136) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:11.6555675Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"PUT","req_id":"ef7cecfc-e4a4-4e6b-94fc-52b8c8e721cd","remote_addr":"[fd00:1122:3344:101::c]:52746","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":591,"response_code":"204"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:11.852978575Z","hostname":"buskin","pid":1555,"instance_id":"ccff4298-547e-428a-b327-016613e4386e","component":"InstanceManager","file":"sled-agent/src/instance.rs:503","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-07-19T17:54:09.156760378Z }, propolis_id: df1fc3a9-614e-45dd-b799-a14257326be0 (propolis), vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-07-19T17:54:11.552693243Z }, migration_state: None }","instance_id":"ccff4298-547e-428a-b327-016613e4386e"} {"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:11.857145623Z","hostname":"buskin","pid":1555,"instance_id":"ccff4298-547e-428a-b327-016613e4386e","component":"InstanceManager","file":"sled-agent/src/instance.rs:736"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:11.892093002Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:61229"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:11.892170124Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"cca1d286-7ffe-4c70-95b2-b278f00b1ae4","remote_addr":"[fd00:1122:3344:101::c]:61229","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":46,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:11.899568197Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"45063872-6458-4f54-8fb8-d67778c5db47","remote_addr":"[fd00:1122:3344:101::c]:61229","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":94,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:11.951436676Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:64693"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:11.951517298Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"360b338e-de3a-4d93-9487-92a9b43b8125","remote_addr":"[fd00:1122:3344:101::c]:64693","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":35,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:11.957883304Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"7b3c2f91-2746-4abf-aed9-258d8597994c","remote_addr":"[fd00:1122:3344:101::c]:64693","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":49,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:12.080709969Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:55162"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:12.081768127Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"8132341b-bb4e-49ff-a3b7-582942909283","remote_addr":"[fd00:1122:3344:101::a]:55162","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":1001,"response_code":"200"} {"msg":"Clearing mapping of virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:12.081991313Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:820","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 248, 48, 19])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(11145136) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:12.082018203Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"DELETE","req_id":"c26b19f2-3ca9-4102-b6bf-9436804bc274","remote_addr":"[fd00:1122:3344:101::a]:55162","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":64,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:12.896563495Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:42032"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:12.896677389Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"9509d037-cc9f-43ef-9495-45d10bd45ba5","remote_addr":"[fd00:1122:3344:101::a]:42032","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":86,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:12.903491336Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"d0cd31b1-1064-407a-9f07-2011c169e60b","remote_addr":"[fd00:1122:3344:101::a]:42032","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":125,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:14.755864659Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:41172"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-19T17:54:14.756007493Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"71729e1f-3a52-4b71-8112-2ecbf6642b80","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:41172","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":95,"response_code":"200"}