[ Oct 12 03:50:44 Enabled. ] [ Oct 12 03:50:44 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ Oct 12 03:50:44 Method "start" exited with status 0. ] [ Oct 12 03:50:44 Rereading configuration. ] [ Oct 12 03:50:44 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:44.130819502Z","hostname":"buskin","pid":1572,"file":"illumos-utils/src/running_zone.rs:113"} {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:44.130958773Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:44.183207572Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:44.183252442Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:44.296250925Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:44.30548576Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:44.313796233Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:44.918258727Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fdb0:18c0:4d0d:9fb2::, width: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:45.05949727Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:126"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:45.059531731Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:138"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:45.059597781Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:151"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:45.059669062Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:520"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:45.059689443Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:173","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:45.059696453Z","hostname":"buskin","pid":1572,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:717"} {"msg":"Generated i86pc baseboard Pc { identifier: \"buskin\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:45.062545288Z","hostname":"buskin","pid":1572,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:624"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:50:45.062551368Z","hostname":"buskin","pid":1572,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:766"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:45.062562048Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:187"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:45.062573809Z","hostname":"buskin","pid":1572,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:825"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:45.062589559Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_0.vdev"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:45.062643839Z","hostname":"buskin","pid":1572,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"None"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:45.06274096Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:45.062769251Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:45.062778611Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:45.074384875Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:45.074418136Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:378","id":"cd56c88b-70fb-4721-a4ca-a4de1b256c20 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:45.173160481Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: cd56c88b-70fb-4721-a4ca-a4de1b256c20 (zpool), kind: Internal }"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:45.574578905Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:45.574610405Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_1.vdev"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:45.574615665Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:45.574627235Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:45.574634555Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:45.574652786Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:45.574657515Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:45.583144571Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:45.583165671Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:378","id":"464bcedf-19e0-4839-a12d-8d477420e762 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:45.669381361Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 464bcedf-19e0-4839-a12d-8d477420e762 (zpool), kind: Internal }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.080518175Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_0.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.080559126Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: cd56c88b-70fb-4721-a4ca-a4de1b256c20 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 464bcedf-19e0-4839-a12d-8d477420e762 (zpool), kind: Internal } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.080600966Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.080617296Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.080624887Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.080631757Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_1.vdev"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.080636217Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.080643067Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.080648307Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.080654037Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: cd56c88b-70fb-4721-a4ca-a4de1b256c20 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 464bcedf-19e0-4839-a12d-8d477420e762 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.080683207Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_2.vdev"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.080687907Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.080692697Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.080697768Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.080703888Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.080708897Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: cd56c88b-70fb-4721-a4ca-a4de1b256c20 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 464bcedf-19e0-4839-a12d-8d477420e762 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.080729028Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.080734168Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.080738128Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_3.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.080742268Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: cd56c88b-70fb-4721-a4ca-a4de1b256c20 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 464bcedf-19e0-4839-a12d-8d477420e762 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.080761538Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.080772988Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.080776969Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.080782169Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.080785298Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_4.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.080788709Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: cd56c88b-70fb-4721-a4ca-a4de1b256c20 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 464bcedf-19e0-4839-a12d-8d477420e762 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.080806739Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.080811839Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.080815709Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.080820789Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.080824619Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_5.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.080828319Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: cd56c88b-70fb-4721-a4ca-a4de1b256c20 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 464bcedf-19e0-4839-a12d-8d477420e762 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.080857219Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.080861819Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.08086919Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.08087318Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.08087704Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_6.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.08088403Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.08088757Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.08089247Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: cd56c88b-70fb-4721-a4ca-a4de1b256c20 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 464bcedf-19e0-4839-a12d-8d477420e762 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.08092717Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_7.vdev"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.08093141Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.080936151Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.080941271Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.0809453Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.080952041Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_8.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.080956411Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.080959721Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.080965241Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: cd56c88b-70fb-4721-a4ca-a4de1b256c20 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 464bcedf-19e0-4839-a12d-8d477420e762 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.081001491Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:97"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.081011861Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:99"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.081015421Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.081019612Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.081022852Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.081026352Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: cd56c88b-70fb-4721-a4ca-a4de1b256c20 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 464bcedf-19e0-4839-a12d-8d477420e762 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.081057812Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.081062602Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.081066012Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:211"} {"msg":"No ledger in /pool/int/cd56c88b-70fb-4721-a4ca-a4de1b256c20/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.081069652Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/464bcedf-19e0-4839-a12d-8d477420e762/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.081088752Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.081094502Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/cd56c88b-70fb-4721-a4ca-a4de1b256c20/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.081110542Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/464bcedf-19e0-4839-a12d-8d477420e762/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.081115153Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.081119333Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.081146213Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:216"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.081163223Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/long_running_tasks.rs:231"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.091974288Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:739"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.092017408Z","hostname":"buskin","pid":1572,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:186","time_to_next_cleanup":"599.999927399s","next_cleanup":"Instant { tv_sec: 5038, tv_nsec: 86874655 }"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.092044078Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3924"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.092062809Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/cd56c88b-70fb-4721-a4ca-a4de1b256c20/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.092125259Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/464bcedf-19e0-4839-a12d-8d477420e762/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.092136979Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.094882284Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:197"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.096517014Z","hostname":"buskin","pid":1572,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:51","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.1.164,\n last: 10.151.1.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.1.164,\n 10.151.1.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.1.100,\n vlan_id: None,\n rib_priority: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.1.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n lldp: None,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n}","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.097102261Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1074"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.097149242Z","hostname":"buskin","pid":1572,"component":"DnsResolver","component":"RSS","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.097235963Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/cd56c88b-70fb-4721-a4ca-a4de1b256c20/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.097278743Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/464bcedf-19e0-4839-a12d-8d477420e762/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.097292343Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.097299643Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.097327244Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/cd56c88b-70fb-4721-a4ca-a4de1b256c20/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.097334964Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/464bcedf-19e0-4839-a12d-8d477420e762/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.097342084Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.097346504Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1179"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.097350194Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:156"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:9fb2::1","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.097360124Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:129"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.097385165Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Writing ledger to /pool/int/cd56c88b-70fb-4721-a4ca-a4de1b256c20/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.097409565Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/464bcedf-19e0-4839-a12d-8d477420e762/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.097726219Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Sled plan written to storage: Plan {\n rack_id: 2b4f7d90-e216-4aab-97d4-74d0f80bf858,\n sleds: {\n [fdb0:18c0:4d0d:9fb2::1]:12346: StartSledAgentRequest {\n generation: 0,\n schema_version: 1,\n body: StartSledAgentRequestBody {\n id: b8a45863-ed9f-4b08-b97f-6fcba7a5a294 (sled),\n rack_id: 2b4f7d90-e216-4aab-97d4-74d0f80bf858,\n use_trust_quorum: false,\n is_lrtq_learner: false,\n subnet: Ipv6Subnet {\n net: Ipv6Net {\n addr: fd00:1122:3344:101::,\n width: 64,\n },\n },\n },\n },\n },\n config: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.1.164,\n last: 10.151.1.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.1.164,\n 10.151.1.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.1.100,\n vlan_id: None,\n rib_priority: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.1.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n lldp: None,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.097991082Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:176"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.098001152Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1213"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.098049963Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/cd56c88b-70fb-4721-a4ca-a4de1b256c20/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.098055903Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/464bcedf-19e0-4839-a12d-8d477420e762/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.098228595Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.098372407Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.098430748Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: b8a45863-ed9f-4b08-b97f-6fcba7a5a294 (sled), rack_id: 2b4f7d90-e216-4aab-97d4-74d0f80bf858, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.099251618Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4221"} {"msg":"Starting switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.099311808Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4603"} {"msg":"Loaded keys and certs","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.10343751Z","hostname":"buskin","pid":1572,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:18c0:4d0d:9fb2::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/client.rs:50"} {"msg":"Certificate chain verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.108868717Z","hostname":"buskin","pid":1572,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:18c0:4d0d:9fb2::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:307"} {"msg":"Signature verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.109055549Z","hostname":"buskin","pid":1572,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:18c0:4d0d:9fb2::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:376"} {"msg":"Certificate chain verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.113971701Z","hostname":"buskin","pid":1572,"component":"SledAgentSprocketsServer","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:307"} {"msg":"Signature verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.114036261Z","hostname":"buskin","pid":1572,"component":"SledAgentSprocketsServer","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:376"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.114167733Z","hostname":"buskin","pid":1572,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:52836","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:74"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: b8a45863-ed9f-4b08-b97f-6fcba7a5a294 (sled), rack_id: 2b4f7d90-e216-4aab-97d4-74d0f80bf858, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.114300654Z","hostname":"buskin","pid":1572,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:362"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.114313095Z","hostname":"buskin","pid":1572,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:377"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.114347785Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/server.rs:44"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.114353455Z","hostname":"buskin","pid":1572,"component":"DnsResolver","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.114363265Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.114367866Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:644"} {"msg":"No ledger in /pool/int/cd56c88b-70fb-4721-a4ca-a4de1b256c20/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.114393946Z","hostname":"buskin","pid":1572,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.114400076Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:101::, width: 64 }"} {"msg":"No ledger in /pool/int/464bcedf-19e0-4839-a12d-8d477420e762/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.114406676Z","hostname":"buskin","pid":1572,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.114418496Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:633"} {"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.114422526Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:661"} {"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.114426486Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:440"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.114444426Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"} {"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.114449007Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:532","result":"DisksManagementResult { status: [] }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.131589769Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.152986035Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.170750675Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.192978621Z","hostname":"buskin","pid":1572,"sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:391"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.193028622Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.193037112Z","hostname":"buskin","pid":1572,"sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:403"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.193079843Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 56.596149ms)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.196226272Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d3ed2470-b1d1-4958-a79c-8e59babcbf7a\", \"content-length\": \"124\", \"date\": \"Sat, 12 Oct 2024 03:50:46 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d3ed2470-b1d1-4958-a79c-8e59babcbf7a\" }"} {"msg":"attempting to create encrypted zvol: name=\"oxi_cd56c88b-70fb-4721-a4ca-a4de1b256c20/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.206874164Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"successfully created encrypted zvol: name=\"oxi_cd56c88b-70fb-4721-a4ca-a4de1b256c20/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.2307051Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_cd56c88b-70fb-4721-a4ca-a4de1b256c20/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.23074665Z","hostname":"buskin","pid":1572,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.236078576Z","hostname":"buskin","pid":1572,"sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:418"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.236091277Z","hostname":"buskin","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:128"} {"msg":"Ensuring dataset oxi_cd56c88b-70fb-4721-a4ca-a4de1b256c20/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.236096006Z","hostname":"buskin","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:138"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.253790456Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 179.409685ms)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.254330713Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f7b99227-1485-4995-98ab-7eb2addeb178\", \"content-length\": \"124\", \"date\": \"Sat, 12 Oct 2024 03:50:46 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f7b99227-1485-4995-98ab-7eb2addeb178\" }"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.32580099Z","hostname":"buskin","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:173"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.434985806Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 142.701466ms)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.435629723Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"792ed1a4-e036-4ba8-a400-f2a9f3a87b2d\", \"content-length\": \"124\", \"date\": \"Sat, 12 Oct 2024 03:50:46 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"792ed1a4-e036-4ba8-a400-f2a9f3a87b2d\" }"} {"msg":"Mounting oxi_cd56c88b-70fb-4721-a4ca-a4de1b256c20/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.544474115Z","hostname":"buskin","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:181"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.557290104Z","hostname":"buskin","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:189"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.557372825Z","hostname":"buskin","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:189"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.557402385Z","hostname":"buskin","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:189"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.557419695Z","hostname":"buskin","pid":1572,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:195"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.579424759Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 112.667236ms)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.579882084Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b432d650-6e05-49ce-a35c-b145d32525e5\", \"content-length\": \"124\", \"date\": \"Sat, 12 Oct 2024 03:50:46 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b432d650-6e05-49ce-a35c-b145d32525e5\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.694034531Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 295.447929ms)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.69472622Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"52da8324-ec19-411e-b7e3-9eceb59c6327\", \"content-length\": \"124\", \"date\": \"Sat, 12 Oct 2024 03:50:46 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"52da8324-ec19-411e-b7e3-9eceb59c6327\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.991310902Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 268.138366ms)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:46.992022191Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"27c05059-65d8-46db-b276-707b60b3571f\", \"content-length\": \"124\", \"date\": \"Sat, 12 Oct 2024 03:50:46 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"27c05059-65d8-46db-b276-707b60b3571f\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:47.261424435Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:47.288606112Z","hostname":"buskin","pid":1572,"sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:93"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:47.294196872Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:34475","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:197","component":"producer-server","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:197"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:47.294235382Z","hostname":"buskin","pid":1572,"component":"internal-dns-resolver","file":"internal-dns/resolver/src/resolver.rs:111","component":"producer-server","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:47.294303663Z","hostname":"buskin","pid":1572,"file":"oximeter/producer/src/lib.rs:283","component":"producer-server","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"oximeter/producer/src/lib.rs:283","interval":"30s","address":"[fd00:1122:3344:101::1]:34475","producer_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:50:47.295244295Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"211.357702ms"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:47.29641992Z","hostname":"buskin","pid":1572,"component":"VmmReservoirManager","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:47.29642922Z","hostname":"buskin","pid":1572,"component":"VmmReservoirManager","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:47.29647671Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:923","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:47.29648235Z","hostname":"buskin","pid":1572,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:50:47.508893547Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"691.148181ms"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:50:48.201813189Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"1.153513931s"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:50:49.356762576Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"2.427039507s"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:49.838220602Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2679"} {"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:49.838246543Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2897"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:49.838256793Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2549"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:49.838260693Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2596"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:49.838264353Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2978"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:49.838267693Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3020"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:49.840105296Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2822"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:49.8412827Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:2590"} {"msg":"Profile for oxz_switch:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:49.841400222Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:49.841609394Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 36.705548ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:50:50.061001038Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 72.289452ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:50:50.251524963Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 147.707997ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:50:50.434995761Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 495.609732ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:50:50.608785718Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 778.616294ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:50:51.149214836Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:50:51.785673967Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"2.82554375s"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.054933774s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:50:52.09683623Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 869.541897ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:50:53.188256558Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.17938722s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:50:54.094556558Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:50:54.61259706Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"4.826019717s"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:55.592296851Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:197"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:55.592354802Z","hostname":"buskin","pid":1572,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:432"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:55.592366472Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Writing ledger to /pool/int/cd56c88b-70fb-4721-a4ca-a4de1b256c20/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:55.592390643Z","hostname":"buskin","pid":1572,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/464bcedf-19e0-4839-a12d-8d477420e762/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:55.592608785Z","hostname":"buskin","pid":1572,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:55.592801528Z","hostname":"buskin","pid":1572,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:55.592846748Z","hostname":"buskin","pid":1572,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV2 { rack_subnet: Ipv6Net { addr: fd00:1122:3344:100::, width: 56 }, infra_ip_first: 10.151.1.174, infra_ip_last: 10.151.1.174, ports: [PortConfigV2 { routes: [RouteConfig { destination: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), nexthop: 10.151.1.100, vlan_id: None, rib_priority: None }], addresses: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false, lldp: None }], bgp: [], bfd: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:55.592861069Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3924"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:55.592932159Z","hostname":"buskin","pid":1572,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:52836","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:81"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:55.84185776Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:117","peer":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:55.841956571Z","hostname":"buskin","pid":1572,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:252","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:55.842090062Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/cd56c88b-70fb-4721-a4ca-a4de1b256c20/config/rss-service-plan-v4.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:55.842183264Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/464bcedf-19e0-4839-a12d-8d477420e762/config/rss-service-plan-v4.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:55.842201484Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:55.842233544Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:55.842263125Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:55.842288135Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:55.847023874Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4245","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:55.847034454Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4261"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:55.847045414Z","hostname":"buskin","pid":1572,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:534"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:56.098252592Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:40696"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:56.098484995Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:56.098491375Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/cd56c88b-70fb-4721-a4ca-a4de1b256c20/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:56.098514926Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/464bcedf-19e0-4839-a12d-8d477420e762/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:56.098530346Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:56.098616917Z","hostname":"buskin","pid":1572,"uri":"/inventory","method":"GET","req_id":"3d925f83-0398-451c-a0b7-74f1f61c6fdf","remote_addr":"[fd00:1122:3344:101::1]:40696","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":212,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:56.129635822Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:55151"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:56.129733723Z","hostname":"buskin","pid":1572,"uri":"/sled-role","method":"GET","req_id":"06e4a464-aead-411a-b96a-4789c3b4c808","remote_addr":"[fd00:1122:3344:101::1]:55151","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":43,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:56.130417431Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Writing ledger to /pool/int/cd56c88b-70fb-4721-a4ca-a4de1b256c20/config/.rss-service-plan-v4.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:56.130442132Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/464bcedf-19e0-4839-a12d-8d477420e762/config/.rss-service-plan-v4.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:56.130739335Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:56.130936858Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:884"} {"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: 8fa52430-429c-4876-b363-73583edd83c9, pool_id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: a4b35157-e976-4581-b3c9-2814a0379398, pool_id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: c3a8b09a-7774-4300-b009-d6793aa3d06c, pool_id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 5f97f87b-a34c-41c3-87e3-273275716110, pool_id: c0c860af-959d-47ac-8156-a6981a984705 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: ed9ae374-a034-46a5-bd01-94890ef83970, pool_id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: 9b5c5f52-1a0f-4c36-a534-25cd9ee805b0, pool_id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: 7c0699af-4384-4561-942a-b626c2cec63f, pool_id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 8ef216c7-80a5-4ef4-abe2-8e35128133fe, pool_id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: 80501b32-98a2-4b36-b612-7f8e50aae318, pool_id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool) }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:56.159856217Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:375"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:56.159965588Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:52158"} {"msg":"physical disks ensure","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:56.160218072Z","hostname":"buskin","pid":1572,"sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:864"} {"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: 8fa52430-429c-4876-b363-73583edd83c9, pool_id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: a4b35157-e976-4581-b3c9-2814a0379398, pool_id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: c3a8b09a-7774-4300-b009-d6793aa3d06c, pool_id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 5f97f87b-a34c-41c3-87e3-273275716110, pool_id: c0c860af-959d-47ac-8156-a6981a984705 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: ed9ae374-a034-46a5-bd01-94890ef83970, pool_id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: 9b5c5f52-1a0f-4c36-a534-25cd9ee805b0, pool_id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: 7c0699af-4384-4561-942a-b626c2cec63f, pool_id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 8ef216c7-80a5-4ef4-abe2-8e35128133fe, pool_id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: 80501b32-98a2-4b36-b612-7f8e50aae318, pool_id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:56.160270812Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/cd56c88b-70fb-4721-a4ca-a4de1b256c20/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:56.160284122Z","hostname":"buskin","pid":1572,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/464bcedf-19e0-4839-a12d-8d477420e762/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:56.160294762Z","hostname":"buskin","pid":1572,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:56.160299522Z","hostname":"buskin","pid":1572,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:866"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:56.160317333Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:56.160322083Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:56.160327943Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:56.160332273Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:56.320792385Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:56.320820495Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool)"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/b8a45863-ed9f-4b08-b97f-6fcba7a5a294)","v":0,"name":"SledAgent","level":50,"time":"2024-10-12T03:50:56.594209309Z","hostname":"buskin","pid":1572,"component":"ProbeManager","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:57.494948111Z","hostname":"buskin","pid":1572,"component":"VmmReservoirManager","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-12T03:50:57.496405709Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"11917948491810414546"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-12T03:50:57.49650558Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"6253452838093016740"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:57.506785658Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:57.516858473Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:57.516955554Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:57.516964814Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:57.516972614Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:57.516980274Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:57.517095636Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:57.517104476Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:57.555783576Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_bed11bbb-e174-4475-bd7f-0211de5b5302/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:57.564201421Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/b8a45863-ed9f-4b08-b97f-6fcba7a5a294)","v":0,"name":"SledAgent","level":50,"time":"2024-10-12T03:50:57.596200968Z","hostname":"buskin","pid":1572,"component":"ProbeManager","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Looking for unencrypted datasets in oxp_bed11bbb-e174-4475-bd7f-0211de5b5302","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:57.742749377Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:57.750979239Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:57.75101144Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:57.75102278Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:57.75102822Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:57.75103368Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:57.760302215Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:57.760318155Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:57.853483512Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:57.86143041Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:57.861465481Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:57.861470981Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:57.861475681Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:57.861481241Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:57.861607222Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:57.861617693Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:57.896753249Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_2b84429f-2c85-4c1f-8f16-c6c336bc6cc2/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:57.903817386Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"Looking for unencrypted datasets in oxp_2b84429f-2c85-4c1f-8f16-c6c336bc6cc2","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.072778043Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.080947044Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.080988525Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.081012075Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.081034705Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.081042316Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.09106735Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.09109019Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.18532091Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","zpool":"ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.193604593Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.193637633Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.193648594Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.193703654Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.193712664Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.193874386Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.193884856Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.230013065Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_ddc852e6-47e6-4d51-a45f-adecc30d1fb3/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.238344538Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"Looking for unencrypted datasets in oxp_ddc852e6-47e6-4d51-a45f-adecc30d1fb3","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.412860025Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.421400071Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.421439051Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.421446371Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.421453331Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.421462922Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.431434045Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.431459386Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"c0c860af-959d-47ac-8156-a6981a984705 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.527784961Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","zpool":"ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.535955223Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.535993493Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.536001223Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.536025784Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.536044624Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.536148515Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.536155255Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.573248126Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_c0c860af-959d-47ac-8156-a6981a984705/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.580650948Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/b8a45863-ed9f-4b08-b97f-6fcba7a5a294)","v":0,"name":"SledAgent","level":50,"time":"2024-10-12T03:50:58.598342857Z","hostname":"buskin","pid":1572,"component":"ProbeManager","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Looking for unencrypted datasets in oxp_c0c860af-959d-47ac-8156-a6981a984705","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.754379184Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.7620783Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.762126871Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.76213278Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.762142221Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.762146631Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.771902172Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.771929252Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.866405705Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","zpool":"ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.874300843Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.874332404Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.874341133Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.874381744Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.874395054Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.874545086Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.874556046Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.910117108Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:58.917850264Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"Looking for unencrypted datasets in oxp_c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.094331244Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.102552936Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.102601946Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.102635177Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.102642717Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.102649817Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.111707839Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.11173331Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"a135191f-1186-46b0-a6c9-d425ec528829 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.205866538Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","zpool":"ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.213819107Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.213856347Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.213866648Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.213908718Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.213921278Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.21405982Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.21407176Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.249720613Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_a135191f-1186-46b0-a6c9-d425ec528829/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.257758903Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"Looking for unencrypted datasets in oxp_a135191f-1186-46b0-a6c9-d425ec528829","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.437067318Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:50:59.44119337Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"17.456587274s"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.445691576Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.445703716Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.445714146Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.445720516Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.445724936Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.455648019Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.45567452Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.555370597Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","zpool":"ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.563417687Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.563444157Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.563451897Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.563513378Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.563526308Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.563625849Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.56363262Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.595402394Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/b8a45863-ed9f-4b08-b97f-6fcba7a5a294)","v":0,"name":"SledAgent","level":50,"time":"2024-10-12T03:50:59.600205764Z","hostname":"buskin","pid":1572,"component":"ProbeManager","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Automatically destroying dataset: oxp_cd705dc8-a605-494a-9c72-bf21f8cc1d40/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.601974176Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"Looking for unencrypted datasets in oxp_cd705dc8-a605-494a-9c72-bf21f8cc1d40","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.770780961Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.777662146Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.777707647Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.777713287Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.777716887Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.777720087Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.788013755Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.788035485Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"42090923-c06f-4776-b57f-20fba0911cf3 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.88262892Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","zpool":"ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.890442377Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.890467587Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.890472687Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.890489007Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.890495397Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.890653819Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.890662679Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.926616256Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_42090923-c06f-4776-b57f-20fba0911cf3/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:50:59.93499529Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"Looking for unencrypted datasets in oxp_42090923-c06f-4776-b57f-20fba0911cf3","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.121850689Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.130599938Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.130640268Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.130684039Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.130692959Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.130708999Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.141204669Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.14122839Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.242011221Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:173","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","zpool":"ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.250440655Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:217","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.250484746Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:219","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.250490956Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:223","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.250496596Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:225","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.250549917Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.250672238Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.250678968Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:237"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.286886018Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_f95ba139-c738-48a5-85f9-774bb88e7e5a/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.295317683Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:280"} {"msg":"Looking for unencrypted datasets in oxp_f95ba139-c738-48a5-85f9-774bb88e7e5a","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.475913405Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/dataset.rs:366"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.484379799Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.48440498Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Writing ledger to /pool/int/cd56c88b-70fb-4721-a4ca-a4de1b256c20/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.484479331Z","hostname":"buskin","pid":1572,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.484604632Z","hostname":"buskin","pid":1572,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: cd56c88b-70fb-4721-a4ca-a4de1b256c20 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 464bcedf-19e0-4839-a12d-8d477420e762 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.484665793Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.484673483Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Writing ledger to /pool/int/464bcedf-19e0-4839-a12d-8d477420e762/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.484800835Z","hostname":"buskin","pid":1572,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"physical disks ensure: Updated storage","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.485056458Z","hostname":"buskin","pid":1572,"sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:868"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.485079878Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"physical disks ensure: Propagating new generation of disks","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.485103099Z","hostname":"buskin","pid":1572,"sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:894","generation":"Generation(1)"} {"msg":"physical disks ensure: Updated storage monitor","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.485109829Z","hostname":"buskin","pid":1572,"sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:899"} {"msg":"physical disks ensure: Updated zone bundler","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.485113989Z","hostname":"buskin","pid":1572,"sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:904"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.485145549Z","hostname":"buskin","pid":1572,"component":"ProbeManager","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:121","gen":"Generation(1)"} {"msg":"physical disks ensure: Updated probes","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.485153229Z","hostname":"buskin","pid":1572,"sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:910"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.485186149Z","hostname":"buskin","pid":1572,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:798","gen":"Generation(1)"} {"msg":"physical disks ensure: Updated instances","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.48519458Z","hostname":"buskin","pid":1572,"sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:915"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.485285151Z","hostname":"buskin","pid":1572,"uri":"/omicron-physical-disks","method":"PUT","req_id":"9a734329-88b1-4315-a5a8-071c0a4c3afb","remote_addr":"[fd00:1122:3344:101::1]:52158","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":4325244,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: d4cfaf42-df38-40c6-be81-9454d6a558a4 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 2767466a-ac74-4716-82bc-b59860a90f48 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5d30c2c5-5827-4b34-ab81-abc29f085544 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.51666651Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.516855053Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:51699"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfaf42-df38-40c6-be81-9454d6a558a4, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.517224227Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.517228177Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4cfaf42-df38-40c6-be81-9454d6a558a4, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2767466a-ac74-4716-82bc-b59860a90f48, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.568043488Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.568068969Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2767466a-ac74-4716-82bc-b59860a90f48, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/b8a45863-ed9f-4b08-b97f-6fcba7a5a294)","v":0,"name":"SledAgent","level":50,"time":"2024-10-12T03:51:00.601846868Z","hostname":"buskin","pid":1572,"component":"ProbeManager","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5d30c2c5-5827-4b34-ab81-abc29f085544, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.618602926Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.618635756Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5d30c2c5-5827-4b34-ab81-abc29f085544, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.669587919Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"No ledger in /pool/int/cd56c88b-70fb-4721-a4ca-a4de1b256c20/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.669622859Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/464bcedf-19e0-4839-a12d-8d477420e762/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.669634579Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.669646359Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.713904319Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.713942419Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.713946169Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_5d30c2c5-5827-4b34-ab81-abc29f085544","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.762022856Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_d4cfaf42-df38-40c6-be81-9454d6a558a4","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.765647391Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_2767466a-ac74-4716-82bc-b59860a90f48","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.769354007Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_internal_dns_5d30c2c5-5827-4b34-ab81-abc29f085544","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.789084742Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_d4cfaf42-df38-40c6-be81-9454d6a558a4","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.79297663Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_2767466a-ac74-4716-82bc-b59860a90f48","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:00.796527714Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/b8a45863-ed9f-4b08-b97f-6fcba7a5a294)","v":0,"name":"SledAgent","level":50,"time":"2024-10-12T03:51:01.603910287Z","hostname":"buskin","pid":1572,"component":"ProbeManager","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Profile for oxz_internal_dns_d4cfaf42-df38-40c6-be81-9454d6a558a4:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:02.345047377Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:02.345099588Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:1::, width: 64 }"} {"msg":"Booting oxz_internal_dns_d4cfaf42-df38-40c6-be81-9454d6a558a4 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:02.345264839Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_d4cfaf42-df38-40c6-be81-9454d6a558a4","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_internal_dns_2767466a-ac74-4716-82bc-b59860a90f48:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:02.828227485Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:02.828280416Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:2::, width: 64 }"} {"msg":"Profile for oxz_internal_dns_5d30c2c5-5827-4b34-ab81-abc29f085544:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:03.310307529Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_internal_dns_d4cfaf42-df38-40c6-be81-9454d6a558a4\") failed: Property not found. retry in 54.886143ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:03.317528008Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_d4cfaf42-df38-40c6-be81-9454d6a558a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_internal_dns_2767466a-ac74-4716-82bc-b59860a90f48 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:03.317591549Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_2767466a-ac74-4716-82bc-b59860a90f48","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:03.321610949Z","hostname":"buskin","pid":1572,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:3::, width: 64 }"} {"msg":"Booting oxz_internal_dns_5d30c2c5-5827-4b34-ab81-abc29f085544 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:03.32167027Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_5d30c2c5-5827-4b34-ab81-abc29f085544","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_internal_dns_d4cfaf42-df38-40c6-be81-9454d6a558a4\") failed: Property not found. retry in 107.516665ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:03.385626924Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_d4cfaf42-df38-40c6-be81-9454d6a558a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_5d30c2c5-5827-4b34-ab81-abc29f085544\") failed: Property not found. retry in 53.096999ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:03.443816466Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_5d30c2c5-5827-4b34-ab81-abc29f085544","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2767466a-ac74-4716-82bc-b59860a90f48\") failed: Property not found. retry in 73.53632ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:03.449917152Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_2767466a-ac74-4716-82bc-b59860a90f48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_d4cfaf42-df38-40c6-be81-9454d6a558a4\") failed: Property not found. retry in 193.372904ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:03.501830976Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_d4cfaf42-df38-40c6-be81-9454d6a558a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_5d30c2c5-5827-4b34-ab81-abc29f085544\") failed: Property not found. retry in 81.109197ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:03.543419053Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_5d30c2c5-5827-4b34-ab81-abc29f085544","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2767466a-ac74-4716-82bc-b59860a90f48\") failed: Property not found. retry in 138.72062ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:03.587260337Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_2767466a-ac74-4716-82bc-b59860a90f48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_5d30c2c5-5827-4b34-ab81-abc29f085544\") failed: Property not found. retry in 141.655546ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:03.64869924Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_5d30c2c5-5827-4b34-ab81-abc29f085544","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_d4cfaf42-df38-40c6-be81-9454d6a558a4\") failed: Property not found. retry in 206.366377ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:03.799861306Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_d4cfaf42-df38-40c6-be81-9454d6a558a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_5d30c2c5-5827-4b34-ab81-abc29f085544\") failed: Property not found. retry in 253.877791ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:03.81872968Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_5d30c2c5-5827-4b34-ab81-abc29f085544","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2767466a-ac74-4716-82bc-b59860a90f48\") failed: Property not found. retry in 140.584447ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:03.848242177Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_2767466a-ac74-4716-82bc-b59860a90f48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2767466a-ac74-4716-82bc-b59860a90f48\") failed: Property not found. retry in 338.843418ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:03.998631554Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_2767466a-ac74-4716-82bc-b59860a90f48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_d4cfaf42-df38-40c6-be81-9454d6a558a4\") failed: Property not found. retry in 740.619908ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:04.013974264Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_d4cfaf42-df38-40c6-be81-9454d6a558a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_5d30c2c5-5827-4b34-ab81-abc29f085544\") failed: Property not found. retry in 1.047705242s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:04.082837479Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_5d30c2c5-5827-4b34-ab81-abc29f085544","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2767466a-ac74-4716-82bc-b59860a90f48\") failed: Property not found. retry in 694.052036ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:04.347488574Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_2767466a-ac74-4716-82bc-b59860a90f48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_d4cfaf42-df38-40c6-be81-9454d6a558a4\") failed: Property not found. retry in 541.857448ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:04.762923191Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_d4cfaf42-df38-40c6-be81-9454d6a558a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2767466a-ac74-4716-82bc-b59860a90f48\") failed: Property not found. retry in 873.942437ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:05.053096323Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_2767466a-ac74-4716-82bc-b59860a90f48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_5d30c2c5-5827-4b34-ab81-abc29f085544\") failed: Property not found. retry in 692.970903ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:05.142050707Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_5d30c2c5-5827-4b34-ab81-abc29f085544","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_d4cfaf42-df38-40c6-be81-9454d6a558a4\") failed: Property not found. retry in 1.303836234s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:05.314218634Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_d4cfaf42-df38-40c6-be81-9454d6a558a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-12T03:51:05.595383004Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5647568185202272368"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-12T03:51:05.595496246Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"3787220255816698279"} {"msg":"wait for service Some(\"oxz_internal_dns_5d30c2c5-5827-4b34-ab81-abc29f085544\") failed: Property not found. retry in 617.157828ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:05.84312414Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_5d30c2c5-5827-4b34-ab81-abc29f085544","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2767466a-ac74-4716-82bc-b59860a90f48\") failed: Property not found. retry in 936.801237ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:05.934659136Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_2767466a-ac74-4716-82bc-b59860a90f48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:06.093786941Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4269"} {"msg":"Ensuring there is a default route","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:06.093830921Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4279","gateway":"fd00:1122:3344:101::1"} {"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:06.099686634Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4358"} {"msg":"refreshed dendrite service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:06.201150574Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4406"} {"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:06.201181534Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4433"} {"msg":"refreshed lldpd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:06.235935355Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4446"} {"msg":"configuring MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:06.235961156Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4310"} {"msg":"refreshed MGS service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:06.286949589Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4352"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:06.286983299Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4413","rack_subnet":"fd00:1122:3344::"} {"msg":"refreshed wicketd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:06.305890174Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4424"} {"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:06.305929414Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4466"} {"msg":"refreshed mgd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:06.372326499Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4498"} {"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:06.372363629Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4504"} {"msg":"refreshed mg-ddm service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:06.460473663Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4545"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:06.460533054Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:370"} {"msg":"wait for service Some(\"oxz_internal_dns_5d30c2c5-5827-4b34-ab81-abc29f085544\") failed: Property not found. retry in 1.354344796s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:06.467854834Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_5d30c2c5-5827-4b34-ab81-abc29f085544","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Failed to get switch ID from MGS (retrying in 43.203049ms)","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:06.496140876Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:389","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id)"} {"msg":"Failed to get switch ID from MGS (retrying in 112.524048ms)","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:06.539601425Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:389","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id)"} {"msg":"wait for service Some(\"oxz_internal_dns_d4cfaf42-df38-40c6-be81-9454d6a558a4\") failed: Property not found. retry in 1.019372496s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:06.625634713Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_d4cfaf42-df38-40c6-be81-9454d6a558a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Failed to get switch ID from MGS (retrying in 201.031877ms)","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:06.652881771Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:389","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id)"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:06.815635852Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::2]:63805"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:06.815896405Z","hostname":"buskin","pid":1572,"uri":"/sled-identifiers","method":"GET","req_id":"afe23e77-bc60-4b20-b39c-50660062d774","remote_addr":"[fd00:1122:3344:101::2]:63805","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":66,"response_code":"200"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:06.855544627Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:418"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:06.887718217Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:686"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:06.887817798Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:724"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:06.89203133Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:727","uptime":"0"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:06.892086011Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:440","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.1.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n}"} {"msg":"wait for service Some(\"oxz_internal_dns_2767466a-ac74-4716-82bc-b59860a90f48\") failed: Property not found. retry in 848.825337ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:06.896578257Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_2767466a-ac74-4716-82bc-b59860a90f48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:06.929691448Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:06.98303491Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:06.983091901Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:06.983112551Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:06.983126181Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:06.983133991Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:06.983158101Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:06.983165142Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:06.983173062Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:06.983179542Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:06.983186452Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:06.983192002Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:06.983197922Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:06.983203542Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:06.983209232Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:06.983215112Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:06.983220902Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:06.983226652Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:06.983236392Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:06.983240843Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:06.983244852Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:06.983248943Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:06.983253543Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:06.983257283Z","hostname":"buskin","pid":1572,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-12T03:51:07.499547542Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"11917948491810414546"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-12T03:51:07.499992757Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"6253452838093016740"} {"msg":"wait for service Some(\"oxz_internal_dns_d4cfaf42-df38-40c6-be81-9454d6a558a4\") failed: Property not found. retry in 548.211173ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:07.653609924Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_d4cfaf42-df38-40c6-be81-9454d6a558a4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_2767466a-ac74-4716-82bc-b59860a90f48\") failed: Property not found. retry in 560.219338ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:07.754669699Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_2767466a-ac74-4716-82bc-b59860a90f48","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_5d30c2c5-5827-4b34-ab81-abc29f085544\") failed: Property not found. retry in 1.02014525s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:07.83046464Z","hostname":"buskin","pid":1572,"zone":"oxz_internal_dns_5d30c2c5-5827-4b34-ab81-abc29f085544","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:08.870772854Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_internal_dns_5d30c2c5-5827-4b34-ab81-abc29f085544"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:08.870820385Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_internal_dns_d4cfaf42-df38-40c6-be81-9454d6a558a4"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:08.870829595Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_internal_dns_2767466a-ac74-4716-82bc-b59860a90f48"} {"msg":"Writing ledger to /pool/int/cd56c88b-70fb-4721-a4ca-a4de1b256c20/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:08.870836315Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/464bcedf-19e0-4839-a12d-8d477420e762/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:08.871181279Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:08.871546734Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"3115291e-3c69-4b00-96d8-7713d088c74f","remote_addr":"[fd00:1122:3344:101::1]:51699","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8354461,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:08.873477658Z","hostname":"buskin","pid":1572,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:08.925143719Z","hostname":"buskin","pid":1572,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:08.969102754Z","hostname":"buskin","pid":1572,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:09.015436839Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:611"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:09.01545261Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:09.04854984Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:09.049912397Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:09.049920107Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: d4cfaf42-df38-40c6-be81-9454d6a558a4 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 2767466a-ac74-4716-82bc-b59860a90f48 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5d30c2c5-5827-4b34-ab81-abc29f085544 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: ac80f25b-dafd-4922-8e9b-8a6b5ae14799 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9341fb54-e534-4734-8c8a-b2833e94a40a, kind: Service { id: ac80f25b-dafd-4922-8e9b-8a6b5ae14799 }, name: Name(\"ntp-ac80f25b-dafd-4922-8e9b-8a6b5ae14799\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 46])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:09.080085632Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:09.080234134Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:63480"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfaf42-df38-40c6-be81-9454d6a558a4, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:09.080420226Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:09.080429296Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4cfaf42-df38-40c6-be81-9454d6a558a4, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2767466a-ac74-4716-82bc-b59860a90f48, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:09.105450287Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:09.105531698Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2767466a-ac74-4716-82bc-b59860a90f48, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5d30c2c5-5827-4b34-ab81-abc29f085544, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:09.125397924Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:09.125439065Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5d30c2c5-5827-4b34-ab81-abc29f085544, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:09.145063679Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/cd56c88b-70fb-4721-a4ca-a4de1b256c20/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:09.145131569Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/464bcedf-19e0-4839-a12d-8d477420e762/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:09.145333052Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:09.145446643Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:09.156797204Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:09.187922371Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:09.188591369Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:09.188595999Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:09.217739981Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 213, 46]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:09.217747431Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1266","zone_type":"boundary_ntp"} {"msg":"no nat entry found for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:09.218119425Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:09.218124866Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:09.218406239Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:09.218781274Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_ntp_ac80f25b-dafd-4922-8e9b-8a6b5ae14799","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:09.244708576Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_ntp_ac80f25b-dafd-4922-8e9b-8a6b5ae14799","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:09.263085744Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_ntp_ac80f25b-dafd-4922-8e9b-8a6b5ae14799:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:09.896575998Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_ntp_ac80f25b-dafd-4922-8e9b-8a6b5ae14799 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:09.896686559Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_ac80f25b-dafd-4922-8e9b-8a6b5ae14799","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_ntp_ac80f25b-dafd-4922-8e9b-8a6b5ae14799\") failed: Property not found. retry in 55.656118ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:10.034257566Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_ac80f25b-dafd-4922-8e9b-8a6b5ae14799","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_ac80f25b-dafd-4922-8e9b-8a6b5ae14799\") failed: Property not found. retry in 60.008886ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:10.10146986Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_ac80f25b-dafd-4922-8e9b-8a6b5ae14799","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_ac80f25b-dafd-4922-8e9b-8a6b5ae14799\") failed: Property not found. retry in 199.296807ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:10.181057458Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_ac80f25b-dafd-4922-8e9b-8a6b5ae14799","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_ac80f25b-dafd-4922-8e9b-8a6b5ae14799\") failed: Property not found. retry in 201.834045ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:10.390106834Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_ac80f25b-dafd-4922-8e9b-8a6b5ae14799","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_ac80f25b-dafd-4922-8e9b-8a6b5ae14799\") failed: Property not found. retry in 1.145050852s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:10.601917593Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_ac80f25b-dafd-4922-8e9b-8a6b5ae14799","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_ac80f25b-dafd-4922-8e9b-8a6b5ae14799\") failed: Property not found. retry in 755.080709ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:11.758560781Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_ac80f25b-dafd-4922-8e9b-8a6b5ae14799","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_ac80f25b-dafd-4922-8e9b-8a6b5ae14799\") failed: Property not found. retry in 965.246272ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:12.522658896Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_ac80f25b-dafd-4922-8e9b-8a6b5ae14799","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_ac80f25b-dafd-4922-8e9b-8a6b5ae14799\") failed: Property not found. retry in 1.188562037s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:13.496028059Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_ac80f25b-dafd-4922-8e9b-8a6b5ae14799","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_ac80f25b-dafd-4922-8e9b-8a6b5ae14799\") failed: Property not found. retry in 1.03272371s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:14.693676237Z","hostname":"buskin","pid":1572,"zone":"oxz_ntp_ac80f25b-dafd-4922-8e9b-8a6b5ae14799","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-12T03:51:15.597099082Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5647568185202272368"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-12T03:51:15.597225474Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"3787220255816698279"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:15.743624391Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_ntp_ac80f25b-dafd-4922-8e9b-8a6b5ae14799"} {"msg":"Writing ledger to /pool/int/cd56c88b-70fb-4721-a4ca-a4de1b256c20/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:15.743679472Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/464bcedf-19e0-4839-a12d-8d477420e762/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:15.744153497Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:15.744530152Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"17ea43f1-2d8e-4bb1-9b8c-47de7fbf58bb","remote_addr":"[fd00:1122:3344:101::1]:63480","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":6664246,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:15.746380365Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:652"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:15.776065274Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:15.776253086Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:37321"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_ac80f25b-dafd-4922-8e9b-8a6b5ae14799': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-10-12T03:51:15.783251883Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3877"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:15.783357794Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"a79137b0-a21d-43f5-8408-6bdb7cdd4f57","remote_addr":"[fd00:1122:3344:101::1]:37321","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":7000,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:15.784746141Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:15.878574916Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:15.878675107Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:43637"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_ac80f25b-dafd-4922-8e9b-8a6b5ae14799': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-10-12T03:51:19.885447046Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3877"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:19.885587067Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"4ddb45c3-d807-434a-9305-66b20b4fa12c","remote_addr":"[fd00:1122:3344:101::1]:43637","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4006743,"response_code":"500"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-12T03:51:19.887168127Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"11917948491810414546"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-12T03:51:19.887369289Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"6253452838093016740"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-12T03:51:19.887522271Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"1270682448713775034"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-12T03:51:19.887673063Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"6384890684953174453"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-10-12T03:51:19.887817945Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"6799942652372860982"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:19.889906881Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:20.015680751Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:20.015832283Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:38321"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_ac80f25b-dafd-4922-8e9b-8a6b5ae14799': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-10-12T03:51:20.023663021Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3877"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:20.023769112Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"db73b8da-c3c8-44c5-ba2b-5273296bae98","remote_addr":"[fd00:1122:3344:101::1]:38321","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":7866,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:20.026890041Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:20.171216982Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:20.171383934Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:52288"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_ac80f25b-dafd-4922-8e9b-8a6b5ae14799': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-10-12T03:51:23.740981627Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3877"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:23.741110729Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"a7b99b7f-8a80-4ea3-89bf-bc8dcad28c71","remote_addr":"[fd00:1122:3344:101::1]:52288","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4009656,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:23.74366535Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:682","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:24.017967763Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:24.018111935Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:49131"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:24.038698611Z","hostname":"buskin","pid":1572,"uri":"/timesync","method":"GET","req_id":"8b9bfac4-adae-4bdd-872e-5cf127c568ff","remote_addr":"[fd00:1122:3344:101::1]:49131","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":20469,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 2180654074, ip_addr: 129.250.35.250, stratum: 3, ref_time: 1728705083.55028, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:24.042884603Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:660"} {"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:24.042891083Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1299"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: d4cfaf42-df38-40c6-be81-9454d6a558a4 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 2767466a-ac74-4716-82bc-b59860a90f48 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5d30c2c5-5827-4b34-ab81-abc29f085544 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: ac80f25b-dafd-4922-8e9b-8a6b5ae14799 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9341fb54-e534-4734-8c8a-b2833e94a40a, kind: Service { id: ac80f25b-dafd-4922-8e9b-8a6b5ae14799 }, name: Name(\"ntp-ac80f25b-dafd-4922-8e9b-8a6b5ae14799\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 46])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0d2c64f-0705-4e92-90c6-5bc73310609a (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ce1b852-d629-4995-8bda-2223e090d747 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0 (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3a86650-54d2-47f3-b719-1e24b48120a2 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:24.011494033Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:24.011647675Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:48254"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfaf42-df38-40c6-be81-9454d6a558a4, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:24.012092981Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:24.012102391Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4cfaf42-df38-40c6-be81-9454d6a558a4, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2767466a-ac74-4716-82bc-b59860a90f48, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:24.032824478Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:24.032864708Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2767466a-ac74-4716-82bc-b59860a90f48, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5d30c2c5-5827-4b34-ab81-abc29f085544, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:24.051325748Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:24.051363178Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5d30c2c5-5827-4b34-ab81-abc29f085544, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0d2c64f-0705-4e92-90c6-5bc73310609a, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:24.069050568Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:24.069087278Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0d2c64f-0705-4e92-90c6-5bc73310609a, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ce1b852-d629-4995-8bda-2223e090d747, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:24.121735892Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:24.121783032Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1ce1b852-d629-4995-8bda-2223e090d747, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:24.172436801Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:24.172474422Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:24.230823736Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:24.230864977Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3a86650-54d2-47f3-b719-1e24b48120a2, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:24.282229864Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:24.282270925Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b3a86650-54d2-47f3-b719-1e24b48120a2, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:24.333830515Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/cd56c88b-70fb-4721-a4ca-a4de1b256c20/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:24.333882295Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/464bcedf-19e0-4839-a12d-8d477420e762/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:24.334067918Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:24.33422961Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:24.415236225Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:24.415271226Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:24.415276806Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:24.415281996Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:24.415286386Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:24.501753168Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_1ce1b852-d629-4995-8bda-2223e090d747","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:24.505503545Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_b3a86650-54d2-47f3-b719-1e24b48120a2","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:24.509518045Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:24.513766168Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_b0d2c64f-0705-4e92-90c6-5bc73310609a","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:24.518051641Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_cockroachdb_4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:24.522332484Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_1ce1b852-d629-4995-8bda-2223e090d747","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:24.526475455Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_b3a86650-54d2-47f3-b719-1e24b48120a2","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:24.531783121Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:24.536265437Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_b0d2c64f-0705-4e92-90c6-5bc73310609a","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:24.540629171Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:27.42921684Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_cockroachdb_4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:27.429355142Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:27.429369642Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_cockroachdb_4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:27.434503566Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_b0d2c64f-0705-4e92-90c6-5bc73310609a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:27.478506834Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_cockroachdb_1ce1b852-d629-4995-8bda-2223e090d747:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:27.478704946Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_b0d2c64f-0705-4e92-90c6-5bc73310609a zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:27.478759877Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_b0d2c64f-0705-4e92-90c6-5bc73310609a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_cockroachdb_1ce1b852-d629-4995-8bda-2223e090d747 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:27.484254895Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_1ce1b852-d629-4995-8bda-2223e090d747","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_b3a86650-54d2-47f3-b719-1e24b48120a2:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:27.48943024Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_b3a86650-54d2-47f3-b719-1e24b48120a2 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:27.489598122Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_b3a86650-54d2-47f3-b719-1e24b48120a2","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0\") failed: Property not found. retry in 33.438398ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:27.572843088Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee\") failed: Property not found. retry in 26.396583ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:27.585717438Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b0d2c64f-0705-4e92-90c6-5bc73310609a\") failed: Property not found. retry in 30.500234ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:27.61076836Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_b0d2c64f-0705-4e92-90c6-5bc73310609a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0\") failed: Property not found. retry in 55.112562ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:27.618195133Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee\") failed: Property not found. retry in 108.948302ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:27.624749844Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1ce1b852-d629-4995-8bda-2223e090d747\") failed: Property not found. retry in 30.509182ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:27.632326539Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_1ce1b852-d629-4995-8bda-2223e090d747","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b3a86650-54d2-47f3-b719-1e24b48120a2\") failed: Property not found. retry in 60.191036ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:27.721964284Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_b3a86650-54d2-47f3-b719-1e24b48120a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1ce1b852-d629-4995-8bda-2223e090d747\") failed: Property not found. retry in 139.751903ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:28.195776272Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_1ce1b852-d629-4995-8bda-2223e090d747","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b3a86650-54d2-47f3-b719-1e24b48120a2\") failed: Property not found. retry in 72.804196ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:28.308771599Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_b3a86650-54d2-47f3-b719-1e24b48120a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0\") failed: Property not found. retry in 297.40844ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:28.532572304Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b0d2c64f-0705-4e92-90c6-5bc73310609a\") failed: Property not found. retry in 57.902151ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:28.587382697Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_b0d2c64f-0705-4e92-90c6-5bc73310609a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee\") failed: Property not found. retry in 229.33352ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:28.987500567Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1ce1b852-d629-4995-8bda-2223e090d747\") failed: Property not found. retry in 288.433464ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:29.171033422Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_1ce1b852-d629-4995-8bda-2223e090d747","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b3a86650-54d2-47f3-b719-1e24b48120a2\") failed: Property not found. retry in 165.483732ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:29.367160633Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_b3a86650-54d2-47f3-b719-1e24b48120a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0\") failed: Property not found. retry in 486.709402ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:30.014873445Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b0d2c64f-0705-4e92-90c6-5bc73310609a\") failed: Property not found. retry in 188.32451ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:30.085767687Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_b0d2c64f-0705-4e92-90c6-5bc73310609a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee\") failed: Property not found. retry in 542.669582ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:30.174436331Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1ce1b852-d629-4995-8bda-2223e090d747\") failed: Property not found. retry in 533.832504ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:30.184073561Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_1ce1b852-d629-4995-8bda-2223e090d747","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b3a86650-54d2-47f3-b719-1e24b48120a2\") failed: Property not found. retry in 250.719916ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:30.194025445Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_b3a86650-54d2-47f3-b719-1e24b48120a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0\") failed: Property not found. retry in 958.020069ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:30.20326609Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b0d2c64f-0705-4e92-90c6-5bc73310609a\") failed: Property not found. retry in 380.652035ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:30.27882812Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_b0d2c64f-0705-4e92-90c6-5bc73310609a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b3a86650-54d2-47f3-b719-1e24b48120a2\") failed: Property not found. retry in 1.18537665s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:30.456011496Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_b3a86650-54d2-47f3-b719-1e24b48120a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b0d2c64f-0705-4e92-90c6-5bc73310609a\") failed: Property not found. retry in 1.175915245s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:30.668500101Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_b0d2c64f-0705-4e92-90c6-5bc73310609a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee\") failed: Property not found. retry in 815.852014ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:30.72700885Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1ce1b852-d629-4995-8bda-2223e090d747\") failed: Property not found. retry in 670.869632ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:30.73506773Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_1ce1b852-d629-4995-8bda-2223e090d747","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0\") failed: Property not found. retry in 1.355087353s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:31.168993181Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1ce1b852-d629-4995-8bda-2223e090d747\") failed: Property not found. retry in 931.914144ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:31.415656592Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_1ce1b852-d629-4995-8bda-2223e090d747","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee\") failed: Property not found. retry in 751.318659ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:31.551764415Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b3a86650-54d2-47f3-b719-1e24b48120a2\") failed: Property not found. retry in 1.134184795s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:31.648238576Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_b3a86650-54d2-47f3-b719-1e24b48120a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b0d2c64f-0705-4e92-90c6-5bc73310609a\") failed: Property not found. retry in 1.453546703s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:31.853275238Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_b0d2c64f-0705-4e92-90c6-5bc73310609a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee\") failed: Property not found. retry in 751.596736ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:32.310962806Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1ce1b852-d629-4995-8bda-2223e090d747\") failed: Property not found. retry in 1.405168791s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:32.35470234Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_1ce1b852-d629-4995-8bda-2223e090d747","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0\") failed: Property not found. retry in 1.239400611s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:32.530503068Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b3a86650-54d2-47f3-b719-1e24b48120a2\") failed: Property not found. retry in 734.530658ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:32.79014252Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_b3a86650-54d2-47f3-b719-1e24b48120a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee\") failed: Property not found. retry in 1.478690789s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:33.069757421Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b0d2c64f-0705-4e92-90c6-5bc73310609a\") failed: Property not found. retry in 1.312751762s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:33.312607464Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_b0d2c64f-0705-4e92-90c6-5bc73310609a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b3a86650-54d2-47f3-b719-1e24b48120a2\") failed: Property not found. retry in 1.320283506s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:33.532917465Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_b3a86650-54d2-47f3-b719-1e24b48120a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1ce1b852-d629-4995-8bda-2223e090d747\") failed: Property not found. retry in 1.280827414s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:33.766699455Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_1ce1b852-d629-4995-8bda-2223e090d747","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0\") failed: Property not found. retry in 576.737159ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:33.77431515Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0\") failed: Property not found. retry in 719.912723ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:34.358257019Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:34.392586666Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"255.147935ms"} {"msg":"wait for service Some(\"oxz_cockroachdb_4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee\") failed: Property not found. retry in 909.733756ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:34.554858636Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b0d2c64f-0705-4e92-90c6-5bc73310609a\") failed: Property not found. retry in 701.80694ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:34.632802166Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_b0d2c64f-0705-4e92-90c6-5bc73310609a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b3a86650-54d2-47f3-b719-1e24b48120a2\") failed: Property not found. retry in 950.432982ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:34.862743578Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_b3a86650-54d2-47f3-b719-1e24b48120a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1ce1b852-d629-4995-8bda-2223e090d747\") failed: Property not found. retry in 1.488222454s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:35.057466132Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_1ce1b852-d629-4995-8bda-2223e090d747","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0\") failed: Property not found. retry in 709.829796ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:35.089092736Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b0d2c64f-0705-4e92-90c6-5bc73310609a\") failed: Property not found. retry in 1.365816723s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:35.342843364Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_b0d2c64f-0705-4e92-90c6-5bc73310609a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee\") failed: Property not found. retry in 626.837155ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:35.471793139Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:35.766717401Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:57483"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:35.766923453Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"e8cd595c-2141-4852-969d-9c0cc59285c7","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:57483","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":74,"response_code":"200"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0\") failed: Property not found. retry in 637.114465ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:35.805108558Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b3a86650-54d2-47f3-b719-1e24b48120a2\") failed: Property not found. retry in 1.277572551s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:35.818568066Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_b3a86650-54d2-47f3-b719-1e24b48120a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee\") failed: Property not found. retry in 631.038167ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:36.105946303Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0\") failed: Property not found. retry in 1.168232489s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:36.450132407Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1ce1b852-d629-4995-8bda-2223e090d747\") failed: Property not found. retry in 803.525971ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:36.550597237Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_1ce1b852-d629-4995-8bda-2223e090d747","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b0d2c64f-0705-4e92-90c6-5bc73310609a\") failed: Property not found. retry in 706.819199ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:36.71388458Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_b0d2c64f-0705-4e92-90c6-5bc73310609a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee\") failed: Property not found. retry in 1.483112456s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:36.743067173Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b3a86650-54d2-47f3-b719-1e24b48120a2\") failed: Property not found. retry in 748.06236ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:37.102398536Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_b3a86650-54d2-47f3-b719-1e24b48120a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1ce1b852-d629-4995-8bda-2223e090d747\") failed: Property not found. retry in 1.108319178s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:37.36142307Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_1ce1b852-d629-4995-8bda-2223e090d747","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b0d2c64f-0705-4e92-90c6-5bc73310609a\") failed: Property not found. retry in 1.167429226s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:37.428431354Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_b0d2c64f-0705-4e92-90c6-5bc73310609a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0\") failed: Property not found. retry in 1.475800858s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:37.625099442Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b3a86650-54d2-47f3-b719-1e24b48120a2\") failed: Property not found. retry in 913.170113ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:37.857709757Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_b3a86650-54d2-47f3-b719-1e24b48120a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee\") failed: Property not found. retry in 1.416279758s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:38.231664512Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1ce1b852-d629-4995-8bda-2223e090d747\") failed: Property not found. retry in 792.453633ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:38.475218034Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_1ce1b852-d629-4995-8bda-2223e090d747","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b0d2c64f-0705-4e92-90c6-5bc73310609a\") failed: Property not found. retry in 825.12851ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:38.601422534Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_b0d2c64f-0705-4e92-90c6-5bc73310609a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b3a86650-54d2-47f3-b719-1e24b48120a2\") failed: Property not found. retry in 743.031049ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:38.776740927Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_b3a86650-54d2-47f3-b719-1e24b48120a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0\") failed: Property not found. retry in 825.950376ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:39.106473031Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_1ce1b852-d629-4995-8bda-2223e090d747\") failed: Property not found. retry in 1.147215697s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:39.273396689Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_1ce1b852-d629-4995-8bda-2223e090d747","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b0d2c64f-0705-4e92-90c6-5bc73310609a\") failed: Property not found. retry in 643.521725ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:39.434283711Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_b0d2c64f-0705-4e92-90c6-5bc73310609a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b3a86650-54d2-47f3-b719-1e24b48120a2\") failed: Property not found. retry in 1.078117474s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:39.526061333Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_b3a86650-54d2-47f3-b719-1e24b48120a2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee\") failed: Property not found. retry in 773.861435ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:39.652997443Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0\") failed: Property not found. retry in 983.595743ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:39.940049216Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0\") failed: Property not found. retry in 1.128113602s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:40.930416243Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0\") failed: Property not found. retry in 558.464024ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:42.065719454Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0\") failed: Property not found. retry in 707.686922ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:42.632452008Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0\") failed: Property not found. retry in 1.456918093s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:43.348190907Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0\") failed: Property not found. retry in 808.738493ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:44.813394094Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0\") failed: Property not found. retry in 1.428397399s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:45.63249543Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0\") failed: Property not found. retry in 634.867011ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:47.067486571Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0\") failed: Property not found. retry in 704.024095ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:47.709609313Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0\") failed: Property not found. retry in 927.243225ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:48.421793257Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0\") failed: Property not found. retry in 914.415559ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:49.354649239Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:49.644271764Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"377.124229ms"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0\") failed: Property not found. retry in 1.269830718s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:50.297866659Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0\") failed: Property not found. retry in 804.369517ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:51.574297187Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0\") failed: Property not found. retry in 970.440389ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:52.386436316Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0\") failed: Property not found. retry in 995.81749ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:53.363503278Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0\") failed: Property not found. retry in 666.298176ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:51:54.365397589Z","hostname":"buskin","pid":1572,"zone":"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:55.05957243Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_cockroachdb_4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:55.05964573Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_cockroachdb_1ce1b852-d629-4995-8bda-2223e090d747"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:55.059655881Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_cockroachdb_b3a86650-54d2-47f3-b719-1e24b48120a2"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:55.059663451Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_cockroachdb_3c626683-6e71-4b7f-a2b1-dc9974cceae0"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:55.059670411Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_cockroachdb_b0d2c64f-0705-4e92-90c6-5bc73310609a"} {"msg":"Writing ledger to /pool/int/cd56c88b-70fb-4721-a4ca-a4de1b256c20/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:55.059692103Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/464bcedf-19e0-4839-a12d-8d477420e762/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:55.070596007Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:55.08210707Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"de2ca384-9bbd-42ce-9342-764ae7c11cd1","remote_addr":"[fd00:1122:3344:101::1]:48254","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":31151647,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:55.11662682Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:40052"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::4]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:55.146229848Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3720"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:51:56.444055522Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3738"} {"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:03.77485394Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3761"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:03.774956991Z","hostname":"buskin","pid":1572,"uri":"/cockroachdb","method":"POST","req_id":"6de9cbcc-defc-47c1-81c6-fce0082e51f1","remote_addr":"[fd00:1122:3344:101::1]:40052","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8681537,"response_code":"204"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d4cfaf42-df38-40c6-be81-9454d6a558a4 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 2767466a-ac74-4716-82bc-b59860a90f48 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5d30c2c5-5827-4b34-ab81-abc29f085544 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: ac80f25b-dafd-4922-8e9b-8a6b5ae14799 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9341fb54-e534-4734-8c8a-b2833e94a40a, kind: Service { id: ac80f25b-dafd-4922-8e9b-8a6b5ae14799 }, name: Name(\"ntp-ac80f25b-dafd-4922-8e9b-8a6b5ae14799\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 46])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0d2c64f-0705-4e92-90c6-5bc73310609a (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ce1b852-d629-4995-8bda-2223e090d747 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0 (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3a86650-54d2-47f3-b719-1e24b48120a2 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: bceffd56-91ce-4f4e-bc98-3823ea9f2972, kind: Service { id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2 }, name: Name(\"external-dns-0e7f7381-564c-4f95-8e8e-4ae9e12efae2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 231])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 5b00adda-efae-4057-bbdc-9e4e9e192aeb, kind: Service { id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b }, name: Name(\"external-dns-3ec2aea5-3e4e-4785-83a2-42c04a52438b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 106])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 4c95767b-a705-4932-acf6-7f257e60acef (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: ce434cc7-d07f-4371-8836-7047706943dd, kind: Service { id: 4c95767b-a705-4932-acf6-7f257e60acef }, name: Name(\"nexus-4c95767b-a705-4932-acf6-7f257e60acef\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 44])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d24f88b5-85d4-41b0-9285-f812ad1f4772 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 704fe666-2d42-412a-9eca-2073bdc12aef, kind: Service { id: d24f88b5-85d4-41b0-9285-f812ad1f4772 }, name: Name(\"nexus-d24f88b5-85d4-41b0-9285-f812ad1f4772\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 98])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 055f90e5-f76f-48a0-93ae-4d52d8b374b4 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: bf4b340c-988d-42cb-b55b-9439e5c74f11, kind: Service { id: 055f90e5-f76f-48a0-93ae-4d52d8b374b4 }, name: Name(\"nexus-055f90e5-f76f-48a0-93ae-4d52d8b374b4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 03484d2d-6ade-48bc-b42f-9afe85526313 (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5db8d812-d71c-4866-80e3-62dcf03b04c2 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 11337050-9a37-49b4-b85c-ccf249a5c075 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 73e5d241-c20b-4f21-a945-5c95b2887c2c (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 185437f9-5949-45df-ad63-9e4a3ae74f66 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: aba8c02d-71d1-44a5-828e-4b07b0810f53 (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781 (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External } } } }, OmicronZoneConfig { id: d3585e5c-c17a-4960-9767-5d4a4b80868c (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External } } } }, OmicronZoneConfig { id: fab91420-061b-4a94-8963-c1358582fd81 (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6762670-2805-43eb-8b0e-62853b8b03ba (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2024f4e2-927b-4e90-b569-39dcf76e856b (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:03.810551285Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:03.810679736Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::1]:43420"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfaf42-df38-40c6-be81-9454d6a558a4, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:03.811098612Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:03.811105852Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4cfaf42-df38-40c6-be81-9454d6a558a4, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2767466a-ac74-4716-82bc-b59860a90f48, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:03.8359149Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:03.835958911Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2767466a-ac74-4716-82bc-b59860a90f48, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5d30c2c5-5827-4b34-ab81-abc29f085544, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:03.857293967Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:03.857347917Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5d30c2c5-5827-4b34-ab81-abc29f085544, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0d2c64f-0705-4e92-90c6-5bc73310609a, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:03.87770303Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:03.877745451Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0d2c64f-0705-4e92-90c6-5bc73310609a, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ce1b852-d629-4995-8bda-2223e090d747, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:03.898304717Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:03.898339247Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1ce1b852-d629-4995-8bda-2223e090d747, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:03.917494196Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:03.917538906Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:03.939003493Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:03.939059274Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3a86650-54d2-47f3-b719-1e24b48120a2, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:03.959045823Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:03.959089594Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b3a86650-54d2-47f3-b719-1e24b48120a2, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:03.978294043Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:03.978335403Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:04.059780537Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:04.059833908Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:04.150288943Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:04.150339934Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 185437f9-5949-45df-ad63-9e4a3ae74f66, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:04.230842756Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:04.230912059Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 185437f9-5949-45df-ad63-9e4a3ae74f66, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:04.370823269Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:04.370884949Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:04.44407213Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:04.444131571Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aba8c02d-71d1-44a5-828e-4b07b0810f53, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:04.517208981Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:04.517261062Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aba8c02d-71d1-44a5-828e-4b07b0810f53, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:04.591590757Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:04.591642277Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d3585e5c-c17a-4960-9767-5d4a4b80868c, dataset_name: DatasetName { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:04.66739743Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:04.667451261Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d3585e5c-c17a-4960-9767-5d4a4b80868c, dataset_name: DatasetName { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fab91420-061b-4a94-8963-c1358582fd81, dataset_name: DatasetName { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:04.735740891Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:04.735799712Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fab91420-061b-4a94-8963-c1358582fd81, dataset_name: DatasetName { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6762670-2805-43eb-8b0e-62853b8b03ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:04.795558826Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:04.795614566Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d6762670-2805-43eb-8b0e-62853b8b03ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2024f4e2-927b-4e90-b569-39dcf76e856b, dataset_name: DatasetName { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:04.861739819Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:04.86179265Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2024f4e2-927b-4e90-b569-39dcf76e856b, dataset_name: DatasetName { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:04.924731453Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/cd56c88b-70fb-4721-a4ca-a4de1b256c20/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:04.924786394Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/464bcedf-19e0-4839-a12d-8d477420e762/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:04.925020267Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:04.925155719Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:05.028238622Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"1.125489132s"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:05.034557961Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:05.034574331Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:05.034580591Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:05.034584871Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:05.034588941Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:05.034592651Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:05.034596491Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:05.034600261Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:05.034605581Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:05.034610661Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:05.034616091Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:05.034622831Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:05.034640942Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:05.034646242Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:05.034651642Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:05.034655262Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:05.034658752Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:05.034662202Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:05.034665652Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.034697142Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"e3ed69af-f4ad-4040-8a0a-0cfdff2c6858","remote_addr":"[fd00:1122:3344:101::1]:43420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c\", TimeNotSynchronized), (\"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2\", TimeNotSynchronized), (\"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781\", TimeNotSynchronized), (\"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b\", TimeNotSynchronized), (\"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66\", TimeNotSynchronized), (\"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23\", TimeNotSynchronized), (\"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb\", TimeNotSynchronized), (\"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb\", TimeNotSynchronized), (\"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53\", TimeNotSynchronized), (\"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772\", TimeNotSynchronized), (\"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313\", TimeNotSynchronized), (\"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81\", TimeNotSynchronized), (\"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba\", TimeNotSynchronized), (\"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef\", TimeNotSynchronized), (\"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2\", TimeNotSynchronized), (\"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075\", TimeNotSynchronized), (\"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4\", TimeNotSynchronized), (\"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c\", TimeNotSynchronized), (\"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b\", TimeNotSynchronized)]","latency_us":1227153,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:05.034859224Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"94.660218ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e3ed69af-f4ad-4040-8a0a-0cfdff2c6858\", \"content-length\": \"124\", \"date\": \"Sat, 12 Oct 2024 03:52:05 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e3ed69af-f4ad-4040-8a0a-0cfdff2c6858\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d4cfaf42-df38-40c6-be81-9454d6a558a4 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 2767466a-ac74-4716-82bc-b59860a90f48 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5d30c2c5-5827-4b34-ab81-abc29f085544 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: ac80f25b-dafd-4922-8e9b-8a6b5ae14799 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9341fb54-e534-4734-8c8a-b2833e94a40a, kind: Service { id: ac80f25b-dafd-4922-8e9b-8a6b5ae14799 }, name: Name(\"ntp-ac80f25b-dafd-4922-8e9b-8a6b5ae14799\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 46])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0d2c64f-0705-4e92-90c6-5bc73310609a (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ce1b852-d629-4995-8bda-2223e090d747 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0 (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3a86650-54d2-47f3-b719-1e24b48120a2 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: bceffd56-91ce-4f4e-bc98-3823ea9f2972, kind: Service { id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2 }, name: Name(\"external-dns-0e7f7381-564c-4f95-8e8e-4ae9e12efae2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 231])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 5b00adda-efae-4057-bbdc-9e4e9e192aeb, kind: Service { id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b }, name: Name(\"external-dns-3ec2aea5-3e4e-4785-83a2-42c04a52438b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 106])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 4c95767b-a705-4932-acf6-7f257e60acef (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: ce434cc7-d07f-4371-8836-7047706943dd, kind: Service { id: 4c95767b-a705-4932-acf6-7f257e60acef }, name: Name(\"nexus-4c95767b-a705-4932-acf6-7f257e60acef\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 44])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d24f88b5-85d4-41b0-9285-f812ad1f4772 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 704fe666-2d42-412a-9eca-2073bdc12aef, kind: Service { id: d24f88b5-85d4-41b0-9285-f812ad1f4772 }, name: Name(\"nexus-d24f88b5-85d4-41b0-9285-f812ad1f4772\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 98])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 055f90e5-f76f-48a0-93ae-4d52d8b374b4 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: bf4b340c-988d-42cb-b55b-9439e5c74f11, kind: Service { id: 055f90e5-f76f-48a0-93ae-4d52d8b374b4 }, name: Name(\"nexus-055f90e5-f76f-48a0-93ae-4d52d8b374b4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 03484d2d-6ade-48bc-b42f-9afe85526313 (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5db8d812-d71c-4866-80e3-62dcf03b04c2 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 11337050-9a37-49b4-b85c-ccf249a5c075 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 73e5d241-c20b-4f21-a945-5c95b2887c2c (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 185437f9-5949-45df-ad63-9e4a3ae74f66 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: aba8c02d-71d1-44a5-828e-4b07b0810f53 (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781 (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External } } } }, OmicronZoneConfig { id: d3585e5c-c17a-4960-9767-5d4a4b80868c (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External } } } }, OmicronZoneConfig { id: fab91420-061b-4a94-8963-c1358582fd81 (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6762670-2805-43eb-8b0e-62853b8b03ba (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2024f4e2-927b-4e90-b569-39dcf76e856b (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.130583586Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfaf42-df38-40c6-be81-9454d6a558a4, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.130876619Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.130883369Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4cfaf42-df38-40c6-be81-9454d6a558a4, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2767466a-ac74-4716-82bc-b59860a90f48, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.154662636Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.154698016Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2767466a-ac74-4716-82bc-b59860a90f48, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5d30c2c5-5827-4b34-ab81-abc29f085544, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.177585291Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.177632181Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5d30c2c5-5827-4b34-ab81-abc29f085544, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0d2c64f-0705-4e92-90c6-5bc73310609a, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.203360992Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.203403632Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0d2c64f-0705-4e92-90c6-5bc73310609a, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ce1b852-d629-4995-8bda-2223e090d747, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.224344463Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.224381023Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1ce1b852-d629-4995-8bda-2223e090d747, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.245110591Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.245146692Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.264731466Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.264770636Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3a86650-54d2-47f3-b719-1e24b48120a2, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.28440272Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.284437071Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b3a86650-54d2-47f3-b719-1e24b48120a2, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.304345449Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.304380489Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.323321485Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.323351685Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.342805217Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.342836848Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 185437f9-5949-45df-ad63-9e4a3ae74f66, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.362159727Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.362192868Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 185437f9-5949-45df-ad63-9e4a3ae74f66, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.381005682Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.381040213Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.401014981Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.401048222Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aba8c02d-71d1-44a5-828e-4b07b0810f53, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.421280263Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.421313034Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aba8c02d-71d1-44a5-828e-4b07b0810f53, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.44187943Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.44191627Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d3585e5c-c17a-4960-9767-5d4a4b80868c, dataset_name: DatasetName { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.461437003Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.461485594Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d3585e5c-c17a-4960-9767-5d4a4b80868c, dataset_name: DatasetName { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fab91420-061b-4a94-8963-c1358582fd81, dataset_name: DatasetName { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.482446835Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.482480065Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fab91420-061b-4a94-8963-c1358582fd81, dataset_name: DatasetName { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6762670-2805-43eb-8b0e-62853b8b03ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.500634291Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.500669892Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d6762670-2805-43eb-8b0e-62853b8b03ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2024f4e2-927b-4e90-b569-39dcf76e856b, dataset_name: DatasetName { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.520848233Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.520880623Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2024f4e2-927b-4e90-b569-39dcf76e856b, dataset_name: DatasetName { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.541610151Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/cd56c88b-70fb-4721-a4ca-a4de1b256c20/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.541649692Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/464bcedf-19e0-4839-a12d-8d477420e762/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.541837994Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.541962165Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:05.642348035Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:05.642402866Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:05.642411546Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:05.642417376Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:05.642422746Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:05.642433166Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:05.642439276Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:05.642451416Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:05.642457076Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:05.642462527Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:05.642469637Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:05.642474717Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:05.642481527Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:05.642486907Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:05.642492327Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:05.642515587Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:05.642519697Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:05.642525137Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:05.642528977Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.642587658Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"c3bc57f3-9851-4000-b115-48c7d7e196d7","remote_addr":"[fd00:1122:3344:101::1]:43420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81\", TimeNotSynchronized), (\"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b\", TimeNotSynchronized), (\"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53\", TimeNotSynchronized), (\"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23\", TimeNotSynchronized), (\"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba\", TimeNotSynchronized), (\"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c\", TimeNotSynchronized), (\"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4\", TimeNotSynchronized), (\"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781\", TimeNotSynchronized), (\"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef\", TimeNotSynchronized), (\"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772\", TimeNotSynchronized), (\"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075\", TimeNotSynchronized), (\"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66\", TimeNotSynchronized), (\"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb\", TimeNotSynchronized), (\"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2\", TimeNotSynchronized), (\"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb\", TimeNotSynchronized), (\"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c\", TimeNotSynchronized), (\"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b\", TimeNotSynchronized), (\"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2\", TimeNotSynchronized), (\"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313\", TimeNotSynchronized)]","latency_us":513266,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:05.64274636Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"151.0486ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c3bc57f3-9851-4000-b115-48c7d7e196d7\", \"content-length\": \"124\", \"date\": \"Sat, 12 Oct 2024 03:52:05 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c3bc57f3-9851-4000-b115-48c7d7e196d7\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.721122836Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:52936"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.721228817Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"8dbf983f-4628-4992-9d19-16c52758b9be","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:52936","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":36,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d4cfaf42-df38-40c6-be81-9454d6a558a4 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 2767466a-ac74-4716-82bc-b59860a90f48 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5d30c2c5-5827-4b34-ab81-abc29f085544 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: ac80f25b-dafd-4922-8e9b-8a6b5ae14799 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9341fb54-e534-4734-8c8a-b2833e94a40a, kind: Service { id: ac80f25b-dafd-4922-8e9b-8a6b5ae14799 }, name: Name(\"ntp-ac80f25b-dafd-4922-8e9b-8a6b5ae14799\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 46])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0d2c64f-0705-4e92-90c6-5bc73310609a (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ce1b852-d629-4995-8bda-2223e090d747 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0 (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3a86650-54d2-47f3-b719-1e24b48120a2 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: bceffd56-91ce-4f4e-bc98-3823ea9f2972, kind: Service { id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2 }, name: Name(\"external-dns-0e7f7381-564c-4f95-8e8e-4ae9e12efae2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 231])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 5b00adda-efae-4057-bbdc-9e4e9e192aeb, kind: Service { id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b }, name: Name(\"external-dns-3ec2aea5-3e4e-4785-83a2-42c04a52438b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 106])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 4c95767b-a705-4932-acf6-7f257e60acef (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: ce434cc7-d07f-4371-8836-7047706943dd, kind: Service { id: 4c95767b-a705-4932-acf6-7f257e60acef }, name: Name(\"nexus-4c95767b-a705-4932-acf6-7f257e60acef\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 44])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d24f88b5-85d4-41b0-9285-f812ad1f4772 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 704fe666-2d42-412a-9eca-2073bdc12aef, kind: Service { id: d24f88b5-85d4-41b0-9285-f812ad1f4772 }, name: Name(\"nexus-d24f88b5-85d4-41b0-9285-f812ad1f4772\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 98])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 055f90e5-f76f-48a0-93ae-4d52d8b374b4 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: bf4b340c-988d-42cb-b55b-9439e5c74f11, kind: Service { id: 055f90e5-f76f-48a0-93ae-4d52d8b374b4 }, name: Name(\"nexus-055f90e5-f76f-48a0-93ae-4d52d8b374b4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 03484d2d-6ade-48bc-b42f-9afe85526313 (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5db8d812-d71c-4866-80e3-62dcf03b04c2 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 11337050-9a37-49b4-b85c-ccf249a5c075 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 73e5d241-c20b-4f21-a945-5c95b2887c2c (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 185437f9-5949-45df-ad63-9e4a3ae74f66 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: aba8c02d-71d1-44a5-828e-4b07b0810f53 (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781 (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External } } } }, OmicronZoneConfig { id: d3585e5c-c17a-4960-9767-5d4a4b80868c (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External } } } }, OmicronZoneConfig { id: fab91420-061b-4a94-8963-c1358582fd81 (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6762670-2805-43eb-8b0e-62853b8b03ba (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2024f4e2-927b-4e90-b569-39dcf76e856b (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.794469599Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfaf42-df38-40c6-be81-9454d6a558a4, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.794776123Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.794784693Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4cfaf42-df38-40c6-be81-9454d6a558a4, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2767466a-ac74-4716-82bc-b59860a90f48, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.817485955Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.817527606Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2767466a-ac74-4716-82bc-b59860a90f48, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5d30c2c5-5827-4b34-ab81-abc29f085544, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.837004608Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.837054619Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5d30c2c5-5827-4b34-ab81-abc29f085544, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0d2c64f-0705-4e92-90c6-5bc73310609a, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.857009987Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.857054478Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0d2c64f-0705-4e92-90c6-5bc73310609a, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ce1b852-d629-4995-8bda-2223e090d747, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.877197018Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.877235989Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1ce1b852-d629-4995-8bda-2223e090d747, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.896957254Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.896998035Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.916977864Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.917022634Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3a86650-54d2-47f3-b719-1e24b48120a2, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.936141922Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.936190863Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b3a86650-54d2-47f3-b719-1e24b48120a2, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.957824232Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.957865182Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.978750813Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.978786913Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.999069775Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:05.999105476Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 185437f9-5949-45df-ad63-9e4a3ae74f66, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.020145338Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.020192788Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 185437f9-5949-45df-ad63-9e4a3ae74f66, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.041689726Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.041727766Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.062273922Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.062311053Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aba8c02d-71d1-44a5-828e-4b07b0810f53, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.082296691Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.082340102Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aba8c02d-71d1-44a5-828e-4b07b0810f53, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.103477085Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.103518275Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d3585e5c-c17a-4960-9767-5d4a4b80868c, dataset_name: DatasetName { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.123602616Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.123655536Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d3585e5c-c17a-4960-9767-5d4a4b80868c, dataset_name: DatasetName { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fab91420-061b-4a94-8963-c1358582fd81, dataset_name: DatasetName { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.143374912Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.143413312Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fab91420-061b-4a94-8963-c1358582fd81, dataset_name: DatasetName { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6762670-2805-43eb-8b0e-62853b8b03ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.164231471Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.164264872Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d6762670-2805-43eb-8b0e-62853b8b03ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2024f4e2-927b-4e90-b569-39dcf76e856b, dataset_name: DatasetName { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.184476043Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.184505354Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2024f4e2-927b-4e90-b569-39dcf76e856b, dataset_name: DatasetName { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.205384414Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/cd56c88b-70fb-4721-a4ca-a4de1b256c20/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.205435104Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/464bcedf-19e0-4839-a12d-8d477420e762/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.205617979Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.205748388Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:06.308289015Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:06.308348774Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:06.308355595Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:06.308362915Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:06.308369105Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:06.308375725Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:06.308381216Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:06.308386696Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:06.308394426Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:06.308400326Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:06.308405706Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:06.308412306Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:06.308417856Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:06.308422106Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:06.308427306Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:06.308449177Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:06.308454946Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:06.308460097Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:06.308463727Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.308509417Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"9916f30a-85cb-4eb0-aa49-e06a2bdd3d37","remote_addr":"[fd00:1122:3344:101::1]:43420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772\", TimeNotSynchronized), (\"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313\", TimeNotSynchronized), (\"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53\", TimeNotSynchronized), (\"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075\", TimeNotSynchronized), (\"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2\", TimeNotSynchronized), (\"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23\", TimeNotSynchronized), (\"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb\", TimeNotSynchronized), (\"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66\", TimeNotSynchronized), (\"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb\", TimeNotSynchronized), (\"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781\", TimeNotSynchronized), (\"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef\", TimeNotSynchronized), (\"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b\", TimeNotSynchronized), (\"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba\", TimeNotSynchronized), (\"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4\", TimeNotSynchronized), (\"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81\", TimeNotSynchronized), (\"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2\", TimeNotSynchronized), (\"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c\", TimeNotSynchronized), (\"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b\", TimeNotSynchronized), (\"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c\", TimeNotSynchronized)]","latency_us":515317,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:06.308694249Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"90.460009ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9916f30a-85cb-4eb0-aa49-e06a2bdd3d37\", \"content-length\": \"124\", \"date\": \"Sat, 12 Oct 2024 03:52:06 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9916f30a-85cb-4eb0-aa49-e06a2bdd3d37\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d4cfaf42-df38-40c6-be81-9454d6a558a4 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 2767466a-ac74-4716-82bc-b59860a90f48 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5d30c2c5-5827-4b34-ab81-abc29f085544 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: ac80f25b-dafd-4922-8e9b-8a6b5ae14799 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9341fb54-e534-4734-8c8a-b2833e94a40a, kind: Service { id: ac80f25b-dafd-4922-8e9b-8a6b5ae14799 }, name: Name(\"ntp-ac80f25b-dafd-4922-8e9b-8a6b5ae14799\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 46])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0d2c64f-0705-4e92-90c6-5bc73310609a (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ce1b852-d629-4995-8bda-2223e090d747 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0 (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3a86650-54d2-47f3-b719-1e24b48120a2 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: bceffd56-91ce-4f4e-bc98-3823ea9f2972, kind: Service { id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2 }, name: Name(\"external-dns-0e7f7381-564c-4f95-8e8e-4ae9e12efae2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 231])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 5b00adda-efae-4057-bbdc-9e4e9e192aeb, kind: Service { id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b }, name: Name(\"external-dns-3ec2aea5-3e4e-4785-83a2-42c04a52438b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 106])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 4c95767b-a705-4932-acf6-7f257e60acef (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: ce434cc7-d07f-4371-8836-7047706943dd, kind: Service { id: 4c95767b-a705-4932-acf6-7f257e60acef }, name: Name(\"nexus-4c95767b-a705-4932-acf6-7f257e60acef\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 44])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d24f88b5-85d4-41b0-9285-f812ad1f4772 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 704fe666-2d42-412a-9eca-2073bdc12aef, kind: Service { id: d24f88b5-85d4-41b0-9285-f812ad1f4772 }, name: Name(\"nexus-d24f88b5-85d4-41b0-9285-f812ad1f4772\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 98])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 055f90e5-f76f-48a0-93ae-4d52d8b374b4 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: bf4b340c-988d-42cb-b55b-9439e5c74f11, kind: Service { id: 055f90e5-f76f-48a0-93ae-4d52d8b374b4 }, name: Name(\"nexus-055f90e5-f76f-48a0-93ae-4d52d8b374b4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 03484d2d-6ade-48bc-b42f-9afe85526313 (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5db8d812-d71c-4866-80e3-62dcf03b04c2 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 11337050-9a37-49b4-b85c-ccf249a5c075 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 73e5d241-c20b-4f21-a945-5c95b2887c2c (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 185437f9-5949-45df-ad63-9e4a3ae74f66 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: aba8c02d-71d1-44a5-828e-4b07b0810f53 (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781 (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External } } } }, OmicronZoneConfig { id: d3585e5c-c17a-4960-9767-5d4a4b80868c (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External } } } }, OmicronZoneConfig { id: fab91420-061b-4a94-8963-c1358582fd81 (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6762670-2805-43eb-8b0e-62853b8b03ba (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2024f4e2-927b-4e90-b569-39dcf76e856b (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.399675912Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfaf42-df38-40c6-be81-9454d6a558a4, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.399982256Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.399988916Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4cfaf42-df38-40c6-be81-9454d6a558a4, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2767466a-ac74-4716-82bc-b59860a90f48, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.422430105Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.422482186Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2767466a-ac74-4716-82bc-b59860a90f48, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5d30c2c5-5827-4b34-ab81-abc29f085544, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.443387336Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.443438437Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5d30c2c5-5827-4b34-ab81-abc29f085544, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0d2c64f-0705-4e92-90c6-5bc73310609a, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.46375844Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.46381152Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0d2c64f-0705-4e92-90c6-5bc73310609a, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ce1b852-d629-4995-8bda-2223e090d747, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.484619479Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.48466242Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1ce1b852-d629-4995-8bda-2223e090d747, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.506887636Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.506937287Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.525786882Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.525825032Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3a86650-54d2-47f3-b719-1e24b48120a2, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.545998373Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.546040734Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b3a86650-54d2-47f3-b719-1e24b48120a2, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.565789Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.56582937Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.585514125Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.585555866Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.607462358Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.607501584Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 185437f9-5949-45df-ad63-9e4a3ae74f66, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.628878115Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.628921706Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 185437f9-5949-45df-ad63-9e4a3ae74f66, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.65176316Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.65181632Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.677214377Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.677262237Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aba8c02d-71d1-44a5-828e-4b07b0810f53, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.701671901Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.701717402Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aba8c02d-71d1-44a5-828e-4b07b0810f53, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.723880568Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.723925308Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d3585e5c-c17a-4960-9767-5d4a4b80868c, dataset_name: DatasetName { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.745369145Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.745414365Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d3585e5c-c17a-4960-9767-5d4a4b80868c, dataset_name: DatasetName { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fab91420-061b-4a94-8963-c1358582fd81, dataset_name: DatasetName { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.765748969Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.765788579Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fab91420-061b-4a94-8963-c1358582fd81, dataset_name: DatasetName { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6762670-2805-43eb-8b0e-62853b8b03ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.786545557Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.786589628Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d6762670-2805-43eb-8b0e-62853b8b03ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2024f4e2-927b-4e90-b569-39dcf76e856b, dataset_name: DatasetName { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.808687303Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.808727564Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2024f4e2-927b-4e90-b569-39dcf76e856b, dataset_name: DatasetName { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.829601973Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/cd56c88b-70fb-4721-a4ca-a4de1b256c20/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.829647734Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/464bcedf-19e0-4839-a12d-8d477420e762/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.829878507Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.830002138Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:06.924995121Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:06.925053712Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:06.925069632Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:06.925076692Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:06.925082112Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:06.925089052Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:06.925092672Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:06.925097722Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:06.925101462Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:06.925105152Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:06.925110542Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:06.925115552Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:06.925120322Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:06.925125022Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:06.925129893Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:06.925134773Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:06.925137943Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:06.925151253Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:06.925156383Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:06.925196963Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"9a77b2cd-86af-4488-881c-2bc7dc634a33","remote_addr":"[fd00:1122:3344:101::1]:43420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772\", TimeNotSynchronized), (\"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313\", TimeNotSynchronized), (\"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2\", TimeNotSynchronized), (\"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66\", TimeNotSynchronized), (\"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781\", TimeNotSynchronized), (\"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b\", TimeNotSynchronized), (\"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53\", TimeNotSynchronized), (\"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2\", TimeNotSynchronized), (\"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb\", TimeNotSynchronized), (\"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb\", TimeNotSynchronized), (\"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81\", TimeNotSynchronized), (\"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c\", TimeNotSynchronized), (\"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4\", TimeNotSynchronized), (\"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075\", TimeNotSynchronized), (\"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba\", TimeNotSynchronized), (\"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef\", TimeNotSynchronized), (\"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23\", TimeNotSynchronized), (\"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c\", TimeNotSynchronized), (\"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b\", TimeNotSynchronized)]","latency_us":526810,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:06.925378856Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"256.238472ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9a77b2cd-86af-4488-881c-2bc7dc634a33\", \"content-length\": \"124\", \"date\": \"Sat, 12 Oct 2024 03:52:06 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9a77b2cd-86af-4488-881c-2bc7dc634a33\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d4cfaf42-df38-40c6-be81-9454d6a558a4 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 2767466a-ac74-4716-82bc-b59860a90f48 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5d30c2c5-5827-4b34-ab81-abc29f085544 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: ac80f25b-dafd-4922-8e9b-8a6b5ae14799 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9341fb54-e534-4734-8c8a-b2833e94a40a, kind: Service { id: ac80f25b-dafd-4922-8e9b-8a6b5ae14799 }, name: Name(\"ntp-ac80f25b-dafd-4922-8e9b-8a6b5ae14799\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 46])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0d2c64f-0705-4e92-90c6-5bc73310609a (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ce1b852-d629-4995-8bda-2223e090d747 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0 (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3a86650-54d2-47f3-b719-1e24b48120a2 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: bceffd56-91ce-4f4e-bc98-3823ea9f2972, kind: Service { id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2 }, name: Name(\"external-dns-0e7f7381-564c-4f95-8e8e-4ae9e12efae2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 231])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 5b00adda-efae-4057-bbdc-9e4e9e192aeb, kind: Service { id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b }, name: Name(\"external-dns-3ec2aea5-3e4e-4785-83a2-42c04a52438b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 106])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 4c95767b-a705-4932-acf6-7f257e60acef (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: ce434cc7-d07f-4371-8836-7047706943dd, kind: Service { id: 4c95767b-a705-4932-acf6-7f257e60acef }, name: Name(\"nexus-4c95767b-a705-4932-acf6-7f257e60acef\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 44])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d24f88b5-85d4-41b0-9285-f812ad1f4772 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 704fe666-2d42-412a-9eca-2073bdc12aef, kind: Service { id: d24f88b5-85d4-41b0-9285-f812ad1f4772 }, name: Name(\"nexus-d24f88b5-85d4-41b0-9285-f812ad1f4772\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 98])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 055f90e5-f76f-48a0-93ae-4d52d8b374b4 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: bf4b340c-988d-42cb-b55b-9439e5c74f11, kind: Service { id: 055f90e5-f76f-48a0-93ae-4d52d8b374b4 }, name: Name(\"nexus-055f90e5-f76f-48a0-93ae-4d52d8b374b4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 03484d2d-6ade-48bc-b42f-9afe85526313 (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5db8d812-d71c-4866-80e3-62dcf03b04c2 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 11337050-9a37-49b4-b85c-ccf249a5c075 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 73e5d241-c20b-4f21-a945-5c95b2887c2c (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 185437f9-5949-45df-ad63-9e4a3ae74f66 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: aba8c02d-71d1-44a5-828e-4b07b0810f53 (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781 (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External } } } }, OmicronZoneConfig { id: d3585e5c-c17a-4960-9767-5d4a4b80868c (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External } } } }, OmicronZoneConfig { id: fab91420-061b-4a94-8963-c1358582fd81 (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6762670-2805-43eb-8b0e-62853b8b03ba (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2024f4e2-927b-4e90-b569-39dcf76e856b (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.182169772Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfaf42-df38-40c6-be81-9454d6a558a4, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.183100924Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.183115114Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4cfaf42-df38-40c6-be81-9454d6a558a4, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2767466a-ac74-4716-82bc-b59860a90f48, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.202559886Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.202599936Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2767466a-ac74-4716-82bc-b59860a90f48, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5d30c2c5-5827-4b34-ab81-abc29f085544, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.225061666Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.225109976Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5d30c2c5-5827-4b34-ab81-abc29f085544, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0d2c64f-0705-4e92-90c6-5bc73310609a, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.245587562Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.245630912Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0d2c64f-0705-4e92-90c6-5bc73310609a, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ce1b852-d629-4995-8bda-2223e090d747, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.263422824Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.263468234Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1ce1b852-d629-4995-8bda-2223e090d747, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.280975282Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.281008012Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.302234836Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.302267537Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3a86650-54d2-47f3-b719-1e24b48120a2, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.32100134Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.321043111Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b3a86650-54d2-47f3-b719-1e24b48120a2, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.339851955Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.339900996Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.358335865Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.358373985Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.379695171Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.379732501Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 185437f9-5949-45df-ad63-9e4a3ae74f66, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.399032571Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.399070122Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 185437f9-5949-45df-ad63-9e4a3ae74f66, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.421129296Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.421174197Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.440193334Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.440239634Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aba8c02d-71d1-44a5-828e-4b07b0810f53, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.459515284Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.459552225Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aba8c02d-71d1-44a5-828e-4b07b0810f53, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.478307648Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.478343039Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d3585e5c-c17a-4960-9767-5d4a4b80868c, dataset_name: DatasetName { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.496888799Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.49692515Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d3585e5c-c17a-4960-9767-5d4a4b80868c, dataset_name: DatasetName { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fab91420-061b-4a94-8963-c1358582fd81, dataset_name: DatasetName { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.516767207Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.516805997Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fab91420-061b-4a94-8963-c1358582fd81, dataset_name: DatasetName { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6762670-2805-43eb-8b0e-62853b8b03ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.538847802Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.538905642Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d6762670-2805-43eb-8b0e-62853b8b03ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2024f4e2-927b-4e90-b569-39dcf76e856b, dataset_name: DatasetName { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.560002075Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.560048725Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2024f4e2-927b-4e90-b569-39dcf76e856b, dataset_name: DatasetName { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.58046333Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/cd56c88b-70fb-4721-a4ca-a4de1b256c20/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.58052151Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/464bcedf-19e0-4839-a12d-8d477420e762/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.580696433Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.580885299Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:07.683980608Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:07.684043029Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:07.684050979Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:07.684058199Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:07.684063549Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:07.684079539Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:07.684085189Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:07.684090349Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:07.68409555Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:07.68410092Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:07.68410605Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:07.68411121Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:07.68411688Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:07.68412202Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:07.68412736Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:07.68413112Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:07.68413637Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:07.68414356Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:07.68414717Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.684234231Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"5dd095ac-c179-4dfd-ba16-e6867ed90e25","remote_addr":"[fd00:1122:3344:101::1]:43420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b\", TimeNotSynchronized), (\"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2\", TimeNotSynchronized), (\"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef\", TimeNotSynchronized), (\"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb\", TimeNotSynchronized), (\"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772\", TimeNotSynchronized), (\"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075\", TimeNotSynchronized), (\"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb\", TimeNotSynchronized), (\"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53\", TimeNotSynchronized), (\"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c\", TimeNotSynchronized), (\"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81\", TimeNotSynchronized), (\"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba\", TimeNotSynchronized), (\"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4\", TimeNotSynchronized), (\"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c\", TimeNotSynchronized), (\"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b\", TimeNotSynchronized), (\"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66\", TimeNotSynchronized), (\"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313\", TimeNotSynchronized), (\"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2\", TimeNotSynchronized), (\"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23\", TimeNotSynchronized), (\"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781\", TimeNotSynchronized)]","latency_us":503301,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:07.684408413Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"146.013442ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5dd095ac-c179-4dfd-ba16-e6867ed90e25\", \"content-length\": \"124\", \"date\": \"Sat, 12 Oct 2024 03:52:07 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5dd095ac-c179-4dfd-ba16-e6867ed90e25\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d4cfaf42-df38-40c6-be81-9454d6a558a4 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 2767466a-ac74-4716-82bc-b59860a90f48 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5d30c2c5-5827-4b34-ab81-abc29f085544 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: ac80f25b-dafd-4922-8e9b-8a6b5ae14799 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9341fb54-e534-4734-8c8a-b2833e94a40a, kind: Service { id: ac80f25b-dafd-4922-8e9b-8a6b5ae14799 }, name: Name(\"ntp-ac80f25b-dafd-4922-8e9b-8a6b5ae14799\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 46])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0d2c64f-0705-4e92-90c6-5bc73310609a (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ce1b852-d629-4995-8bda-2223e090d747 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0 (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3a86650-54d2-47f3-b719-1e24b48120a2 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: bceffd56-91ce-4f4e-bc98-3823ea9f2972, kind: Service { id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2 }, name: Name(\"external-dns-0e7f7381-564c-4f95-8e8e-4ae9e12efae2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 231])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 5b00adda-efae-4057-bbdc-9e4e9e192aeb, kind: Service { id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b }, name: Name(\"external-dns-3ec2aea5-3e4e-4785-83a2-42c04a52438b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 106])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 4c95767b-a705-4932-acf6-7f257e60acef (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: ce434cc7-d07f-4371-8836-7047706943dd, kind: Service { id: 4c95767b-a705-4932-acf6-7f257e60acef }, name: Name(\"nexus-4c95767b-a705-4932-acf6-7f257e60acef\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 44])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d24f88b5-85d4-41b0-9285-f812ad1f4772 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 704fe666-2d42-412a-9eca-2073bdc12aef, kind: Service { id: d24f88b5-85d4-41b0-9285-f812ad1f4772 }, name: Name(\"nexus-d24f88b5-85d4-41b0-9285-f812ad1f4772\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 98])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 055f90e5-f76f-48a0-93ae-4d52d8b374b4 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: bf4b340c-988d-42cb-b55b-9439e5c74f11, kind: Service { id: 055f90e5-f76f-48a0-93ae-4d52d8b374b4 }, name: Name(\"nexus-055f90e5-f76f-48a0-93ae-4d52d8b374b4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 03484d2d-6ade-48bc-b42f-9afe85526313 (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5db8d812-d71c-4866-80e3-62dcf03b04c2 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 11337050-9a37-49b4-b85c-ccf249a5c075 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 73e5d241-c20b-4f21-a945-5c95b2887c2c (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 185437f9-5949-45df-ad63-9e4a3ae74f66 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: aba8c02d-71d1-44a5-828e-4b07b0810f53 (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781 (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External } } } }, OmicronZoneConfig { id: d3585e5c-c17a-4960-9767-5d4a4b80868c (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External } } } }, OmicronZoneConfig { id: fab91420-061b-4a94-8963-c1358582fd81 (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6762670-2805-43eb-8b0e-62853b8b03ba (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2024f4e2-927b-4e90-b569-39dcf76e856b (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.831397073Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfaf42-df38-40c6-be81-9454d6a558a4, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.831725837Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.831737057Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4cfaf42-df38-40c6-be81-9454d6a558a4, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2767466a-ac74-4716-82bc-b59860a90f48, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.855358791Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.855400542Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2767466a-ac74-4716-82bc-b59860a90f48, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5d30c2c5-5827-4b34-ab81-abc29f085544, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.87769573Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.87773948Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5d30c2c5-5827-4b34-ab81-abc29f085544, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0d2c64f-0705-4e92-90c6-5bc73310609a, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.899563132Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.899609492Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0d2c64f-0705-4e92-90c6-5bc73310609a, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ce1b852-d629-4995-8bda-2223e090d747, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.921711497Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.921753468Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1ce1b852-d629-4995-8bda-2223e090d747, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.942757699Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.94279812Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.964915885Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.964957296Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3a86650-54d2-47f3-b719-1e24b48120a2, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.98781148Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:07.987853481Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b3a86650-54d2-47f3-b719-1e24b48120a2, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.011289582Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.011331033Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.033352167Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.033411468Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.056463915Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.056509275Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 185437f9-5949-45df-ad63-9e4a3ae74f66, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.078644391Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.078684441Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 185437f9-5949-45df-ad63-9e4a3ae74f66, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.101152711Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.101197781Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.123484919Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.12354305Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aba8c02d-71d1-44a5-828e-4b07b0810f53, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.147430047Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.147479497Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aba8c02d-71d1-44a5-828e-4b07b0810f53, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.170280243Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.170321503Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d3585e5c-c17a-4960-9767-5d4a4b80868c, dataset_name: DatasetName { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.193821494Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.193864465Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d3585e5c-c17a-4960-9767-5d4a4b80868c, dataset_name: DatasetName { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fab91420-061b-4a94-8963-c1358582fd81, dataset_name: DatasetName { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.214858416Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.214903867Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fab91420-061b-4a94-8963-c1358582fd81, dataset_name: DatasetName { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6762670-2805-43eb-8b0e-62853b8b03ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.236457305Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.236496066Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d6762670-2805-43eb-8b0e-62853b8b03ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2024f4e2-927b-4e90-b569-39dcf76e856b, dataset_name: DatasetName { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.258131465Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.258175015Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2024f4e2-927b-4e90-b569-39dcf76e856b, dataset_name: DatasetName { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.280381112Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/cd56c88b-70fb-4721-a4ca-a4de1b256c20/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.280436403Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/464bcedf-19e0-4839-a12d-8d477420e762/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.280636495Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.280773547Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:08.384919712Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:08.384975023Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:08.384982753Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:08.384989773Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:08.384995153Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:08.385001963Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:08.385005743Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:08.385010624Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:08.385026574Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:08.385033164Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:08.385036804Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:08.385041804Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:08.385045554Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:08.385049474Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:08.385056054Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:08.385059514Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:08.385065924Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:08.385070674Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:08.385075674Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.385134355Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"7cb9622f-95b0-402e-b6d9-099f4df505f9","remote_addr":"[fd00:1122:3344:101::1]:43420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23\", TimeNotSynchronized), (\"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2\", TimeNotSynchronized), (\"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb\", TimeNotSynchronized), (\"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772\", TimeNotSynchronized), (\"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b\", TimeNotSynchronized), (\"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313\", TimeNotSynchronized), (\"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4\", TimeNotSynchronized), (\"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c\", TimeNotSynchronized), (\"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81\", TimeNotSynchronized), (\"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b\", TimeNotSynchronized), (\"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53\", TimeNotSynchronized), (\"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66\", TimeNotSynchronized), (\"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba\", TimeNotSynchronized), (\"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c\", TimeNotSynchronized), (\"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781\", TimeNotSynchronized), (\"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2\", TimeNotSynchronized), (\"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef\", TimeNotSynchronized), (\"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075\", TimeNotSynchronized), (\"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb\", TimeNotSynchronized)]","latency_us":555106,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:08.385304377Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"256.610407ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7cb9622f-95b0-402e-b6d9-099f4df505f9\", \"content-length\": \"124\", \"date\": \"Sat, 12 Oct 2024 03:52:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7cb9622f-95b0-402e-b6d9-099f4df505f9\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d4cfaf42-df38-40c6-be81-9454d6a558a4 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 2767466a-ac74-4716-82bc-b59860a90f48 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5d30c2c5-5827-4b34-ab81-abc29f085544 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: ac80f25b-dafd-4922-8e9b-8a6b5ae14799 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9341fb54-e534-4734-8c8a-b2833e94a40a, kind: Service { id: ac80f25b-dafd-4922-8e9b-8a6b5ae14799 }, name: Name(\"ntp-ac80f25b-dafd-4922-8e9b-8a6b5ae14799\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 46])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0d2c64f-0705-4e92-90c6-5bc73310609a (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ce1b852-d629-4995-8bda-2223e090d747 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0 (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3a86650-54d2-47f3-b719-1e24b48120a2 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: bceffd56-91ce-4f4e-bc98-3823ea9f2972, kind: Service { id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2 }, name: Name(\"external-dns-0e7f7381-564c-4f95-8e8e-4ae9e12efae2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 231])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 5b00adda-efae-4057-bbdc-9e4e9e192aeb, kind: Service { id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b }, name: Name(\"external-dns-3ec2aea5-3e4e-4785-83a2-42c04a52438b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 106])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 4c95767b-a705-4932-acf6-7f257e60acef (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: ce434cc7-d07f-4371-8836-7047706943dd, kind: Service { id: 4c95767b-a705-4932-acf6-7f257e60acef }, name: Name(\"nexus-4c95767b-a705-4932-acf6-7f257e60acef\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 44])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d24f88b5-85d4-41b0-9285-f812ad1f4772 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 704fe666-2d42-412a-9eca-2073bdc12aef, kind: Service { id: d24f88b5-85d4-41b0-9285-f812ad1f4772 }, name: Name(\"nexus-d24f88b5-85d4-41b0-9285-f812ad1f4772\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 98])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 055f90e5-f76f-48a0-93ae-4d52d8b374b4 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: bf4b340c-988d-42cb-b55b-9439e5c74f11, kind: Service { id: 055f90e5-f76f-48a0-93ae-4d52d8b374b4 }, name: Name(\"nexus-055f90e5-f76f-48a0-93ae-4d52d8b374b4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 03484d2d-6ade-48bc-b42f-9afe85526313 (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5db8d812-d71c-4866-80e3-62dcf03b04c2 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 11337050-9a37-49b4-b85c-ccf249a5c075 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 73e5d241-c20b-4f21-a945-5c95b2887c2c (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 185437f9-5949-45df-ad63-9e4a3ae74f66 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: aba8c02d-71d1-44a5-828e-4b07b0810f53 (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781 (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External } } } }, OmicronZoneConfig { id: d3585e5c-c17a-4960-9767-5d4a4b80868c (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External } } } }, OmicronZoneConfig { id: fab91420-061b-4a94-8963-c1358582fd81 (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6762670-2805-43eb-8b0e-62853b8b03ba (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2024f4e2-927b-4e90-b569-39dcf76e856b (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.641942422Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfaf42-df38-40c6-be81-9454d6a558a4, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.642271226Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.642279936Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4cfaf42-df38-40c6-be81-9454d6a558a4, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2767466a-ac74-4716-82bc-b59860a90f48, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.665202271Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.665251472Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2767466a-ac74-4716-82bc-b59860a90f48, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5d30c2c5-5827-4b34-ab81-abc29f085544, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.687496079Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.687539369Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5d30c2c5-5827-4b34-ab81-abc29f085544, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0d2c64f-0705-4e92-90c6-5bc73310609a, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.708424589Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.7084698Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0d2c64f-0705-4e92-90c6-5bc73310609a, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ce1b852-d629-4995-8bda-2223e090d747, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.730773081Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.730812232Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1ce1b852-d629-4995-8bda-2223e090d747, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.752989564Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.753035874Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.774463901Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.774500162Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3a86650-54d2-47f3-b719-1e24b48120a2, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.795480093Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.795514793Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b3a86650-54d2-47f3-b719-1e24b48120a2, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.816536385Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.816576665Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.839350059Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.839385859Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.861885079Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.86193548Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 185437f9-5949-45df-ad63-9e4a3ae74f66, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.882718969Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.882764909Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 185437f9-5949-45df-ad63-9e4a3ae74f66, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.903159343Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.903195654Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.926152779Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.92618958Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aba8c02d-71d1-44a5-828e-4b07b0810f53, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.946584954Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.946621024Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aba8c02d-71d1-44a5-828e-4b07b0810f53, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.967740307Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.967775467Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d3585e5c-c17a-4960-9767-5d4a4b80868c, dataset_name: DatasetName { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.989572779Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:08.989609479Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d3585e5c-c17a-4960-9767-5d4a4b80868c, dataset_name: DatasetName { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fab91420-061b-4a94-8963-c1358582fd81, dataset_name: DatasetName { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.011633083Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.011676764Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fab91420-061b-4a94-8963-c1358582fd81, dataset_name: DatasetName { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6762670-2805-43eb-8b0e-62853b8b03ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.034815902Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.034855502Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d6762670-2805-43eb-8b0e-62853b8b03ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2024f4e2-927b-4e90-b569-39dcf76e856b, dataset_name: DatasetName { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.055630751Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.055684302Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2024f4e2-927b-4e90-b569-39dcf76e856b, dataset_name: DatasetName { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.077436552Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/cd56c88b-70fb-4721-a4ca-a4de1b256c20/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.077488573Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/464bcedf-19e0-4839-a12d-8d477420e762/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.077699866Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.077820967Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:09.183188249Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:09.1832488Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:09.18325739Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:09.18326456Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:09.18327135Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:09.18327715Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:09.18328446Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:09.18329035Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:09.18329721Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:09.18330259Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:09.18331949Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:09.183325001Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:09.183331701Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:09.183335681Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:09.183342591Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:09.183347861Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:09.183354841Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:09.183360041Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:09.183366821Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.183465582Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"e2103c8b-c575-49f9-b423-b234105abcc8","remote_addr":"[fd00:1122:3344:101::1]:43420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba\", TimeNotSynchronized), (\"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53\", TimeNotSynchronized), (\"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313\", TimeNotSynchronized), (\"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2\", TimeNotSynchronized), (\"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772\", TimeNotSynchronized), (\"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075\", TimeNotSynchronized), (\"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2\", TimeNotSynchronized), (\"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c\", TimeNotSynchronized), (\"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef\", TimeNotSynchronized), (\"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb\", TimeNotSynchronized), (\"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66\", TimeNotSynchronized), (\"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4\", TimeNotSynchronized), (\"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b\", TimeNotSynchronized), (\"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23\", TimeNotSynchronized), (\"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781\", TimeNotSynchronized), (\"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb\", TimeNotSynchronized), (\"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c\", TimeNotSynchronized), (\"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b\", TimeNotSynchronized), (\"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81\", TimeNotSynchronized)]","latency_us":542842,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:09.183629204Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"348.810089ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e2103c8b-c575-49f9-b423-b234105abcc8\", \"content-length\": \"124\", \"date\": \"Sat, 12 Oct 2024 03:52:09 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e2103c8b-c575-49f9-b423-b234105abcc8\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d4cfaf42-df38-40c6-be81-9454d6a558a4 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 2767466a-ac74-4716-82bc-b59860a90f48 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5d30c2c5-5827-4b34-ab81-abc29f085544 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: ac80f25b-dafd-4922-8e9b-8a6b5ae14799 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9341fb54-e534-4734-8c8a-b2833e94a40a, kind: Service { id: ac80f25b-dafd-4922-8e9b-8a6b5ae14799 }, name: Name(\"ntp-ac80f25b-dafd-4922-8e9b-8a6b5ae14799\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 46])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0d2c64f-0705-4e92-90c6-5bc73310609a (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ce1b852-d629-4995-8bda-2223e090d747 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0 (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3a86650-54d2-47f3-b719-1e24b48120a2 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: bceffd56-91ce-4f4e-bc98-3823ea9f2972, kind: Service { id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2 }, name: Name(\"external-dns-0e7f7381-564c-4f95-8e8e-4ae9e12efae2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 231])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 5b00adda-efae-4057-bbdc-9e4e9e192aeb, kind: Service { id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b }, name: Name(\"external-dns-3ec2aea5-3e4e-4785-83a2-42c04a52438b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 106])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 4c95767b-a705-4932-acf6-7f257e60acef (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: ce434cc7-d07f-4371-8836-7047706943dd, kind: Service { id: 4c95767b-a705-4932-acf6-7f257e60acef }, name: Name(\"nexus-4c95767b-a705-4932-acf6-7f257e60acef\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 44])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d24f88b5-85d4-41b0-9285-f812ad1f4772 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 704fe666-2d42-412a-9eca-2073bdc12aef, kind: Service { id: d24f88b5-85d4-41b0-9285-f812ad1f4772 }, name: Name(\"nexus-d24f88b5-85d4-41b0-9285-f812ad1f4772\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 98])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 055f90e5-f76f-48a0-93ae-4d52d8b374b4 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: bf4b340c-988d-42cb-b55b-9439e5c74f11, kind: Service { id: 055f90e5-f76f-48a0-93ae-4d52d8b374b4 }, name: Name(\"nexus-055f90e5-f76f-48a0-93ae-4d52d8b374b4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 03484d2d-6ade-48bc-b42f-9afe85526313 (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5db8d812-d71c-4866-80e3-62dcf03b04c2 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 11337050-9a37-49b4-b85c-ccf249a5c075 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 73e5d241-c20b-4f21-a945-5c95b2887c2c (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 185437f9-5949-45df-ad63-9e4a3ae74f66 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: aba8c02d-71d1-44a5-828e-4b07b0810f53 (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781 (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External } } } }, OmicronZoneConfig { id: d3585e5c-c17a-4960-9767-5d4a4b80868c (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External } } } }, OmicronZoneConfig { id: fab91420-061b-4a94-8963-c1358582fd81 (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6762670-2805-43eb-8b0e-62853b8b03ba (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2024f4e2-927b-4e90-b569-39dcf76e856b (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.533116375Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfaf42-df38-40c6-be81-9454d6a558a4, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.533447499Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.533454369Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4cfaf42-df38-40c6-be81-9454d6a558a4, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2767466a-ac74-4716-82bc-b59860a90f48, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.556462625Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.556515216Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2767466a-ac74-4716-82bc-b59860a90f48, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5d30c2c5-5827-4b34-ab81-abc29f085544, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.580351543Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.580397483Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5d30c2c5-5827-4b34-ab81-abc29f085544, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0d2c64f-0705-4e92-90c6-5bc73310609a, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.601614727Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.601666318Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0d2c64f-0705-4e92-90c6-5bc73310609a, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ce1b852-d629-4995-8bda-2223e090d747, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.62429893Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.62434415Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1ce1b852-d629-4995-8bda-2223e090d747, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.645600895Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.645644855Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.666571716Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.666623526Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3a86650-54d2-47f3-b719-1e24b48120a2, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.686867098Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.686914679Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b3a86650-54d2-47f3-b719-1e24b48120a2, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.708586909Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.708636899Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.730632363Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.730673434Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.752414334Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.752455555Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 185437f9-5949-45df-ad63-9e4a3ae74f66, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.774380268Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.774427378Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 185437f9-5949-45df-ad63-9e4a3ae74f66, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.798053492Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.798097443Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.820649403Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.820686014Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aba8c02d-71d1-44a5-828e-4b07b0810f53, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.842587717Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.842621327Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aba8c02d-71d1-44a5-828e-4b07b0810f53, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.864099544Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.864131905Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d3585e5c-c17a-4960-9767-5d4a4b80868c, dataset_name: DatasetName { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.88623452Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.88626814Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d3585e5c-c17a-4960-9767-5d4a4b80868c, dataset_name: DatasetName { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fab91420-061b-4a94-8963-c1358582fd81, dataset_name: DatasetName { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.907452084Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.907493795Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fab91420-061b-4a94-8963-c1358582fd81, dataset_name: DatasetName { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6762670-2805-43eb-8b0e-62853b8b03ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.929896734Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.929929974Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d6762670-2805-43eb-8b0e-62853b8b03ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2024f4e2-927b-4e90-b569-39dcf76e856b, dataset_name: DatasetName { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.952182191Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.952214211Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2024f4e2-927b-4e90-b569-39dcf76e856b, dataset_name: DatasetName { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.972873648Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/cd56c88b-70fb-4721-a4ca-a4de1b256c20/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.972913499Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/464bcedf-19e0-4839-a12d-8d477420e762/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.973116781Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:09.973244633Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:10.078833057Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:10.078894338Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:10.078902208Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:10.078909188Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:10.078914368Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:10.078921368Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:10.078926958Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:10.078933898Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:10.078939219Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:10.078945859Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:10.078951669Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:10.078958129Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:10.078963239Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:10.078978209Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:10.078983889Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:10.078990749Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:10.078997449Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:10.079001449Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:10.07900695Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:10.079114171Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"10a1974f-3475-4a11-bb9b-2417df4e66a3","remote_addr":"[fd00:1122:3344:101::1]:43420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075\", TimeNotSynchronized), (\"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772\", TimeNotSynchronized), (\"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c\", TimeNotSynchronized), (\"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66\", TimeNotSynchronized), (\"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2\", TimeNotSynchronized), (\"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c\", TimeNotSynchronized), (\"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b\", TimeNotSynchronized), (\"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81\", TimeNotSynchronized), (\"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781\", TimeNotSynchronized), (\"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53\", TimeNotSynchronized), (\"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba\", TimeNotSynchronized), (\"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23\", TimeNotSynchronized), (\"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb\", TimeNotSynchronized), (\"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef\", TimeNotSynchronized), (\"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4\", TimeNotSynchronized), (\"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b\", TimeNotSynchronized), (\"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313\", TimeNotSynchronized), (\"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2\", TimeNotSynchronized), (\"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb\", TimeNotSynchronized)]","latency_us":547339,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:10.079278343Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"432.552298ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"10a1974f-3475-4a11-bb9b-2417df4e66a3\", \"content-length\": \"124\", \"date\": \"Sat, 12 Oct 2024 03:52:09 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"10a1974f-3475-4a11-bb9b-2417df4e66a3\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d4cfaf42-df38-40c6-be81-9454d6a558a4 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 2767466a-ac74-4716-82bc-b59860a90f48 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5d30c2c5-5827-4b34-ab81-abc29f085544 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: ac80f25b-dafd-4922-8e9b-8a6b5ae14799 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9341fb54-e534-4734-8c8a-b2833e94a40a, kind: Service { id: ac80f25b-dafd-4922-8e9b-8a6b5ae14799 }, name: Name(\"ntp-ac80f25b-dafd-4922-8e9b-8a6b5ae14799\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 46])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0d2c64f-0705-4e92-90c6-5bc73310609a (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ce1b852-d629-4995-8bda-2223e090d747 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0 (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3a86650-54d2-47f3-b719-1e24b48120a2 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: bceffd56-91ce-4f4e-bc98-3823ea9f2972, kind: Service { id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2 }, name: Name(\"external-dns-0e7f7381-564c-4f95-8e8e-4ae9e12efae2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 231])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 5b00adda-efae-4057-bbdc-9e4e9e192aeb, kind: Service { id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b }, name: Name(\"external-dns-3ec2aea5-3e4e-4785-83a2-42c04a52438b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 106])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 4c95767b-a705-4932-acf6-7f257e60acef (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: ce434cc7-d07f-4371-8836-7047706943dd, kind: Service { id: 4c95767b-a705-4932-acf6-7f257e60acef }, name: Name(\"nexus-4c95767b-a705-4932-acf6-7f257e60acef\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 44])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d24f88b5-85d4-41b0-9285-f812ad1f4772 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 704fe666-2d42-412a-9eca-2073bdc12aef, kind: Service { id: d24f88b5-85d4-41b0-9285-f812ad1f4772 }, name: Name(\"nexus-d24f88b5-85d4-41b0-9285-f812ad1f4772\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 98])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 055f90e5-f76f-48a0-93ae-4d52d8b374b4 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: bf4b340c-988d-42cb-b55b-9439e5c74f11, kind: Service { id: 055f90e5-f76f-48a0-93ae-4d52d8b374b4 }, name: Name(\"nexus-055f90e5-f76f-48a0-93ae-4d52d8b374b4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 03484d2d-6ade-48bc-b42f-9afe85526313 (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5db8d812-d71c-4866-80e3-62dcf03b04c2 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 11337050-9a37-49b4-b85c-ccf249a5c075 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 73e5d241-c20b-4f21-a945-5c95b2887c2c (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 185437f9-5949-45df-ad63-9e4a3ae74f66 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: aba8c02d-71d1-44a5-828e-4b07b0810f53 (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781 (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External } } } }, OmicronZoneConfig { id: d3585e5c-c17a-4960-9767-5d4a4b80868c (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External } } } }, OmicronZoneConfig { id: fab91420-061b-4a94-8963-c1358582fd81 (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6762670-2805-43eb-8b0e-62853b8b03ba (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2024f4e2-927b-4e90-b569-39dcf76e856b (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:10.512432593Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfaf42-df38-40c6-be81-9454d6a558a4, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:10.512746797Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:10.512755417Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4cfaf42-df38-40c6-be81-9454d6a558a4, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2767466a-ac74-4716-82bc-b59860a90f48, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:10.536517583Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:10.536565564Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2767466a-ac74-4716-82bc-b59860a90f48, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5d30c2c5-5827-4b34-ab81-abc29f085544, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:10.560026966Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:10.560072757Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5d30c2c5-5827-4b34-ab81-abc29f085544, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0d2c64f-0705-4e92-90c6-5bc73310609a, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:10.580833295Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:10.580881185Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0d2c64f-0705-4e92-90c6-5bc73310609a, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ce1b852-d629-4995-8bda-2223e090d747, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:10.601135918Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:10.601182528Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1ce1b852-d629-4995-8bda-2223e090d747, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:10.624547489Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:10.62458885Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:10.643993321Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:10.644035242Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3a86650-54d2-47f3-b719-1e24b48120a2, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:10.665248696Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:10.665298046Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b3a86650-54d2-47f3-b719-1e24b48120a2, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:10.684719168Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:10.684760359Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:10.705891282Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:10.705933822Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:10.725232672Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:10.725275913Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 185437f9-5949-45df-ad63-9e4a3ae74f66, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:10.746302325Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:10.746359715Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 185437f9-5949-45df-ad63-9e4a3ae74f66, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:10.766383355Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:10.766424055Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:10.787479707Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:10.787521588Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aba8c02d-71d1-44a5-828e-4b07b0810f53, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:10.807969782Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:10.808018313Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aba8c02d-71d1-44a5-828e-4b07b0810f53, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:10.827739918Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:10.827781329Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d3585e5c-c17a-4960-9767-5d4a4b80868c, dataset_name: DatasetName { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:10.849185255Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:10.849227506Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d3585e5c-c17a-4960-9767-5d4a4b80868c, dataset_name: DatasetName { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fab91420-061b-4a94-8963-c1358582fd81, dataset_name: DatasetName { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:10.870145466Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:10.870188897Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fab91420-061b-4a94-8963-c1358582fd81, dataset_name: DatasetName { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6762670-2805-43eb-8b0e-62853b8b03ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:10.890632441Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:10.890674122Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d6762670-2805-43eb-8b0e-62853b8b03ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2024f4e2-927b-4e90-b569-39dcf76e856b, dataset_name: DatasetName { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:10.912032047Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:10.912077148Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2024f4e2-927b-4e90-b569-39dcf76e856b, dataset_name: DatasetName { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:10.932384091Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/cd56c88b-70fb-4721-a4ca-a4de1b256c20/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:10.932433601Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/464bcedf-19e0-4839-a12d-8d477420e762/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:10.932636094Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:10.932833377Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:11.033744763Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:11.033828144Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:11.033834784Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:11.033840544Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:11.033847454Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:11.033852774Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:11.033858444Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:11.033862084Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:11.033865504Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:11.033869004Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:11.033873804Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:11.033877624Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:11.033882544Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:11.033886014Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:11.033889564Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:11.033893064Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:11.033902784Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:11.033907925Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:11.033911345Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:11.033916555Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"5aff9299-f8b2-4131-85ad-fa8b8f57e4a8","remote_addr":"[fd00:1122:3344:101::1]:43420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81\", TimeNotSynchronized), (\"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b\", TimeNotSynchronized), (\"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef\", TimeNotSynchronized), (\"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4\", TimeNotSynchronized), (\"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075\", TimeNotSynchronized), (\"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23\", TimeNotSynchronized), (\"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b\", TimeNotSynchronized), (\"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313\", TimeNotSynchronized), (\"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53\", TimeNotSynchronized), (\"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781\", TimeNotSynchronized), (\"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2\", TimeNotSynchronized), (\"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772\", TimeNotSynchronized), (\"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb\", TimeNotSynchronized), (\"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c\", TimeNotSynchronized), (\"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb\", TimeNotSynchronized), (\"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66\", TimeNotSynchronized), (\"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c\", TimeNotSynchronized), (\"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2\", TimeNotSynchronized), (\"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba\", TimeNotSynchronized)]","latency_us":522768,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:11.034089257Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"638.784485ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5aff9299-f8b2-4131-85ad-fa8b8f57e4a8\", \"content-length\": \"124\", \"date\": \"Sat, 12 Oct 2024 03:52:11 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5aff9299-f8b2-4131-85ad-fa8b8f57e4a8\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d4cfaf42-df38-40c6-be81-9454d6a558a4 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 2767466a-ac74-4716-82bc-b59860a90f48 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5d30c2c5-5827-4b34-ab81-abc29f085544 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: ac80f25b-dafd-4922-8e9b-8a6b5ae14799 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9341fb54-e534-4734-8c8a-b2833e94a40a, kind: Service { id: ac80f25b-dafd-4922-8e9b-8a6b5ae14799 }, name: Name(\"ntp-ac80f25b-dafd-4922-8e9b-8a6b5ae14799\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 46])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0d2c64f-0705-4e92-90c6-5bc73310609a (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ce1b852-d629-4995-8bda-2223e090d747 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0 (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3a86650-54d2-47f3-b719-1e24b48120a2 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: bceffd56-91ce-4f4e-bc98-3823ea9f2972, kind: Service { id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2 }, name: Name(\"external-dns-0e7f7381-564c-4f95-8e8e-4ae9e12efae2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 231])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 5b00adda-efae-4057-bbdc-9e4e9e192aeb, kind: Service { id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b }, name: Name(\"external-dns-3ec2aea5-3e4e-4785-83a2-42c04a52438b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 106])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 4c95767b-a705-4932-acf6-7f257e60acef (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: ce434cc7-d07f-4371-8836-7047706943dd, kind: Service { id: 4c95767b-a705-4932-acf6-7f257e60acef }, name: Name(\"nexus-4c95767b-a705-4932-acf6-7f257e60acef\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 44])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d24f88b5-85d4-41b0-9285-f812ad1f4772 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 704fe666-2d42-412a-9eca-2073bdc12aef, kind: Service { id: d24f88b5-85d4-41b0-9285-f812ad1f4772 }, name: Name(\"nexus-d24f88b5-85d4-41b0-9285-f812ad1f4772\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 98])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 055f90e5-f76f-48a0-93ae-4d52d8b374b4 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: bf4b340c-988d-42cb-b55b-9439e5c74f11, kind: Service { id: 055f90e5-f76f-48a0-93ae-4d52d8b374b4 }, name: Name(\"nexus-055f90e5-f76f-48a0-93ae-4d52d8b374b4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 03484d2d-6ade-48bc-b42f-9afe85526313 (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5db8d812-d71c-4866-80e3-62dcf03b04c2 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 11337050-9a37-49b4-b85c-ccf249a5c075 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 73e5d241-c20b-4f21-a945-5c95b2887c2c (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 185437f9-5949-45df-ad63-9e4a3ae74f66 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: aba8c02d-71d1-44a5-828e-4b07b0810f53 (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781 (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External } } } }, OmicronZoneConfig { id: d3585e5c-c17a-4960-9767-5d4a4b80868c (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External } } } }, OmicronZoneConfig { id: fab91420-061b-4a94-8963-c1358582fd81 (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6762670-2805-43eb-8b0e-62853b8b03ba (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2024f4e2-927b-4e90-b569-39dcf76e856b (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:11.672004047Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfaf42-df38-40c6-be81-9454d6a558a4, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:11.672988889Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:11.67300889Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4cfaf42-df38-40c6-be81-9454d6a558a4, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2767466a-ac74-4716-82bc-b59860a90f48, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:11.695682392Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:11.695736662Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2767466a-ac74-4716-82bc-b59860a90f48, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5d30c2c5-5827-4b34-ab81-abc29f085544, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:11.716555052Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:11.716603502Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5d30c2c5-5827-4b34-ab81-abc29f085544, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0d2c64f-0705-4e92-90c6-5bc73310609a, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:11.737248539Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:11.73731007Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0d2c64f-0705-4e92-90c6-5bc73310609a, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ce1b852-d629-4995-8bda-2223e090d747, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:11.760706061Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:11.760749112Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1ce1b852-d629-4995-8bda-2223e090d747, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:11.781706123Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:11.781749803Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:11.802965127Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:11.803012398Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3a86650-54d2-47f3-b719-1e24b48120a2, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:11.822714193Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:11.822755444Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b3a86650-54d2-47f3-b719-1e24b48120a2, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:11.842972065Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:11.843023266Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:11.866956984Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:11.867000764Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:11.887476189Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:11.88751519Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 185437f9-5949-45df-ad63-9e4a3ae74f66, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:11.908511661Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:11.908552251Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 185437f9-5949-45df-ad63-9e4a3ae74f66, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:11.929247189Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:11.92929171Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:11.950738567Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:11.950779837Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aba8c02d-71d1-44a5-828e-4b07b0810f53, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:11.969984856Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:11.970024981Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aba8c02d-71d1-44a5-828e-4b07b0810f53, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:11.990803315Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:11.990843046Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d3585e5c-c17a-4960-9767-5d4a4b80868c, dataset_name: DatasetName { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:12.009448247Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:12.009489188Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d3585e5c-c17a-4960-9767-5d4a4b80868c, dataset_name: DatasetName { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fab91420-061b-4a94-8963-c1358582fd81, dataset_name: DatasetName { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:12.030036774Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:12.030082224Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fab91420-061b-4a94-8963-c1358582fd81, dataset_name: DatasetName { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6762670-2805-43eb-8b0e-62853b8b03ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:12.050067303Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:12.050111754Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d6762670-2805-43eb-8b0e-62853b8b03ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2024f4e2-927b-4e90-b569-39dcf76e856b, dataset_name: DatasetName { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:12.070337885Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:12.070390566Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2024f4e2-927b-4e90-b569-39dcf76e856b, dataset_name: DatasetName { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:12.090402475Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/cd56c88b-70fb-4721-a4ca-a4de1b256c20/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:12.090452656Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/464bcedf-19e0-4839-a12d-8d477420e762/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:12.090626848Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:12.090734573Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:12.18877145Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:12.18883609Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:12.18884586Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:12.188871231Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:12.188878261Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:12.188885191Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:12.188890701Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:12.188897141Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:12.188903941Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:12.188909531Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:12.188916541Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:12.188922081Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:12.188927252Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:12.188933202Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:12.188936882Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:12.188940542Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:12.188945882Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:12.188952402Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:12.188957802Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:12.189001622Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"4c4c9f13-74f8-4b84-aa79-8be4d269b36c","remote_addr":"[fd00:1122:3344:101::1]:43420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b\", TimeNotSynchronized), (\"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb\", TimeNotSynchronized), (\"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef\", TimeNotSynchronized), (\"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313\", TimeNotSynchronized), (\"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23\", TimeNotSynchronized), (\"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772\", TimeNotSynchronized), (\"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2\", TimeNotSynchronized), (\"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53\", TimeNotSynchronized), (\"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c\", TimeNotSynchronized), (\"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb\", TimeNotSynchronized), (\"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66\", TimeNotSynchronized), (\"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c\", TimeNotSynchronized), (\"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4\", TimeNotSynchronized), (\"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba\", TimeNotSynchronized), (\"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075\", TimeNotSynchronized), (\"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2\", TimeNotSynchronized), (\"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b\", TimeNotSynchronized), (\"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81\", TimeNotSynchronized), (\"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781\", TimeNotSynchronized)]","latency_us":518303,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:12.189168464Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"576.109276ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4c4c9f13-74f8-4b84-aa79-8be4d269b36c\", \"content-length\": \"124\", \"date\": \"Sat, 12 Oct 2024 03:52:11 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4c4c9f13-74f8-4b84-aa79-8be4d269b36c\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d4cfaf42-df38-40c6-be81-9454d6a558a4 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 2767466a-ac74-4716-82bc-b59860a90f48 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5d30c2c5-5827-4b34-ab81-abc29f085544 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: ac80f25b-dafd-4922-8e9b-8a6b5ae14799 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9341fb54-e534-4734-8c8a-b2833e94a40a, kind: Service { id: ac80f25b-dafd-4922-8e9b-8a6b5ae14799 }, name: Name(\"ntp-ac80f25b-dafd-4922-8e9b-8a6b5ae14799\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 46])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0d2c64f-0705-4e92-90c6-5bc73310609a (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ce1b852-d629-4995-8bda-2223e090d747 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0 (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3a86650-54d2-47f3-b719-1e24b48120a2 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: bceffd56-91ce-4f4e-bc98-3823ea9f2972, kind: Service { id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2 }, name: Name(\"external-dns-0e7f7381-564c-4f95-8e8e-4ae9e12efae2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 231])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 5b00adda-efae-4057-bbdc-9e4e9e192aeb, kind: Service { id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b }, name: Name(\"external-dns-3ec2aea5-3e4e-4785-83a2-42c04a52438b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 106])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 4c95767b-a705-4932-acf6-7f257e60acef (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: ce434cc7-d07f-4371-8836-7047706943dd, kind: Service { id: 4c95767b-a705-4932-acf6-7f257e60acef }, name: Name(\"nexus-4c95767b-a705-4932-acf6-7f257e60acef\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 44])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d24f88b5-85d4-41b0-9285-f812ad1f4772 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 704fe666-2d42-412a-9eca-2073bdc12aef, kind: Service { id: d24f88b5-85d4-41b0-9285-f812ad1f4772 }, name: Name(\"nexus-d24f88b5-85d4-41b0-9285-f812ad1f4772\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 98])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 055f90e5-f76f-48a0-93ae-4d52d8b374b4 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: bf4b340c-988d-42cb-b55b-9439e5c74f11, kind: Service { id: 055f90e5-f76f-48a0-93ae-4d52d8b374b4 }, name: Name(\"nexus-055f90e5-f76f-48a0-93ae-4d52d8b374b4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 03484d2d-6ade-48bc-b42f-9afe85526313 (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5db8d812-d71c-4866-80e3-62dcf03b04c2 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 11337050-9a37-49b4-b85c-ccf249a5c075 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 73e5d241-c20b-4f21-a945-5c95b2887c2c (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 185437f9-5949-45df-ad63-9e4a3ae74f66 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: aba8c02d-71d1-44a5-828e-4b07b0810f53 (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781 (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External } } } }, OmicronZoneConfig { id: d3585e5c-c17a-4960-9767-5d4a4b80868c (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External } } } }, OmicronZoneConfig { id: fab91420-061b-4a94-8963-c1358582fd81 (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6762670-2805-43eb-8b0e-62853b8b03ba (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2024f4e2-927b-4e90-b569-39dcf76e856b (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:12.765221224Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfaf42-df38-40c6-be81-9454d6a558a4, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:12.765532888Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:12.765540358Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4cfaf42-df38-40c6-be81-9454d6a558a4, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2767466a-ac74-4716-82bc-b59860a90f48, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:12.788888129Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:12.788938269Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2767466a-ac74-4716-82bc-b59860a90f48, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5d30c2c5-5827-4b34-ab81-abc29f085544, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:12.812226139Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:12.81226644Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5d30c2c5-5827-4b34-ab81-abc29f085544, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0d2c64f-0705-4e92-90c6-5bc73310609a, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:12.833432623Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:12.833477754Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0d2c64f-0705-4e92-90c6-5bc73310609a, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ce1b852-d629-4995-8bda-2223e090d747, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:12.854601947Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:12.854659459Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1ce1b852-d629-4995-8bda-2223e090d747, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:12.876912535Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:12.876951195Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:12.896753822Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:12.896794352Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3a86650-54d2-47f3-b719-1e24b48120a2, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:12.917079724Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:12.917121595Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b3a86650-54d2-47f3-b719-1e24b48120a2, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:12.938091226Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:12.938125916Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:12.960048469Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:12.96008687Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:12.98019339Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:12.980233541Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 185437f9-5949-45df-ad63-9e4a3ae74f66, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:13.00187577Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:13.00193026Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 185437f9-5949-45df-ad63-9e4a3ae74f66, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:13.026243273Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:13.026291934Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:13.047832952Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:13.047878843Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aba8c02d-71d1-44a5-828e-4b07b0810f53, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:13.069191128Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:13.069230048Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aba8c02d-71d1-44a5-828e-4b07b0810f53, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:13.090707466Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:13.090748236Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d3585e5c-c17a-4960-9767-5d4a4b80868c, dataset_name: DatasetName { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:13.112684189Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:13.11272786Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d3585e5c-c17a-4960-9767-5d4a4b80868c, dataset_name: DatasetName { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fab91420-061b-4a94-8963-c1358582fd81, dataset_name: DatasetName { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:13.133824642Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:13.133864493Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fab91420-061b-4a94-8963-c1358582fd81, dataset_name: DatasetName { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6762670-2805-43eb-8b0e-62853b8b03ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:13.155407871Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:13.155454082Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d6762670-2805-43eb-8b0e-62853b8b03ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2024f4e2-927b-4e90-b569-39dcf76e856b, dataset_name: DatasetName { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:13.176123569Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:13.176160189Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2024f4e2-927b-4e90-b569-39dcf76e856b, dataset_name: DatasetName { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:13.197579716Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/cd56c88b-70fb-4721-a4ca-a4de1b256c20/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:13.197631647Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/464bcedf-19e0-4839-a12d-8d477420e762/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:13.197838829Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:13.197971401Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:13.300043221Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:13.300101952Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:13.300110002Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:13.300117452Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:13.300121312Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:13.300126372Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:13.300150853Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:13.300155683Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:13.300160073Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:13.300165973Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:13.300170213Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:13.300175893Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:13.300180043Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:13.300184513Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:13.300188793Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:13.300194823Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:13.300199023Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:13.300203083Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:13.300207233Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:13.300211453Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"23750221-64f8-49d2-9621-78d4c37c18a9","remote_addr":"[fd00:1122:3344:101::1]:43420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313\", TimeNotSynchronized), (\"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2\", TimeNotSynchronized), (\"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb\", TimeNotSynchronized), (\"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53\", TimeNotSynchronized), (\"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef\", TimeNotSynchronized), (\"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c\", TimeNotSynchronized), (\"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba\", TimeNotSynchronized), (\"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2\", TimeNotSynchronized), (\"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b\", TimeNotSynchronized), (\"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075\", TimeNotSynchronized), (\"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb\", TimeNotSynchronized), (\"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81\", TimeNotSynchronized), (\"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772\", TimeNotSynchronized), (\"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4\", TimeNotSynchronized), (\"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66\", TimeNotSynchronized), (\"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781\", TimeNotSynchronized), (\"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23\", TimeNotSynchronized), (\"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b\", TimeNotSynchronized), (\"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c\", TimeNotSynchronized)]","latency_us":536320,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:13.300359795Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"744.185576ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"23750221-64f8-49d2-9621-78d4c37c18a9\", \"content-length\": \"124\", \"date\": \"Sat, 12 Oct 2024 03:52:12 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"23750221-64f8-49d2-9621-78d4c37c18a9\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d4cfaf42-df38-40c6-be81-9454d6a558a4 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 2767466a-ac74-4716-82bc-b59860a90f48 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5d30c2c5-5827-4b34-ab81-abc29f085544 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: ac80f25b-dafd-4922-8e9b-8a6b5ae14799 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9341fb54-e534-4734-8c8a-b2833e94a40a, kind: Service { id: ac80f25b-dafd-4922-8e9b-8a6b5ae14799 }, name: Name(\"ntp-ac80f25b-dafd-4922-8e9b-8a6b5ae14799\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 46])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0d2c64f-0705-4e92-90c6-5bc73310609a (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ce1b852-d629-4995-8bda-2223e090d747 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0 (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3a86650-54d2-47f3-b719-1e24b48120a2 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: bceffd56-91ce-4f4e-bc98-3823ea9f2972, kind: Service { id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2 }, name: Name(\"external-dns-0e7f7381-564c-4f95-8e8e-4ae9e12efae2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 231])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 5b00adda-efae-4057-bbdc-9e4e9e192aeb, kind: Service { id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b }, name: Name(\"external-dns-3ec2aea5-3e4e-4785-83a2-42c04a52438b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 106])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 4c95767b-a705-4932-acf6-7f257e60acef (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: ce434cc7-d07f-4371-8836-7047706943dd, kind: Service { id: 4c95767b-a705-4932-acf6-7f257e60acef }, name: Name(\"nexus-4c95767b-a705-4932-acf6-7f257e60acef\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 44])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d24f88b5-85d4-41b0-9285-f812ad1f4772 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 704fe666-2d42-412a-9eca-2073bdc12aef, kind: Service { id: d24f88b5-85d4-41b0-9285-f812ad1f4772 }, name: Name(\"nexus-d24f88b5-85d4-41b0-9285-f812ad1f4772\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 98])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 055f90e5-f76f-48a0-93ae-4d52d8b374b4 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: bf4b340c-988d-42cb-b55b-9439e5c74f11, kind: Service { id: 055f90e5-f76f-48a0-93ae-4d52d8b374b4 }, name: Name(\"nexus-055f90e5-f76f-48a0-93ae-4d52d8b374b4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 03484d2d-6ade-48bc-b42f-9afe85526313 (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5db8d812-d71c-4866-80e3-62dcf03b04c2 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 11337050-9a37-49b4-b85c-ccf249a5c075 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 73e5d241-c20b-4f21-a945-5c95b2887c2c (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 185437f9-5949-45df-ad63-9e4a3ae74f66 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: aba8c02d-71d1-44a5-828e-4b07b0810f53 (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781 (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External } } } }, OmicronZoneConfig { id: d3585e5c-c17a-4960-9767-5d4a4b80868c (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External } } } }, OmicronZoneConfig { id: fab91420-061b-4a94-8963-c1358582fd81 (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6762670-2805-43eb-8b0e-62853b8b03ba (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2024f4e2-927b-4e90-b569-39dcf76e856b (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:14.04389732Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfaf42-df38-40c6-be81-9454d6a558a4, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:14.044169923Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:14.044185283Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4cfaf42-df38-40c6-be81-9454d6a558a4, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2767466a-ac74-4716-82bc-b59860a90f48, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:14.065845483Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:14.065877073Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2767466a-ac74-4716-82bc-b59860a90f48, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5d30c2c5-5827-4b34-ab81-abc29f085544, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:14.088723727Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:14.088757568Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5d30c2c5-5827-4b34-ab81-abc29f085544, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0d2c64f-0705-4e92-90c6-5bc73310609a, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:14.107515201Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:14.107541862Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0d2c64f-0705-4e92-90c6-5bc73310609a, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ce1b852-d629-4995-8bda-2223e090d747, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:14.127937095Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:14.127965246Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1ce1b852-d629-4995-8bda-2223e090d747, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:14.149672636Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:14.149716477Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:14.170143821Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:14.170172291Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3a86650-54d2-47f3-b719-1e24b48120a2, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:14.191607878Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:14.191641548Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b3a86650-54d2-47f3-b719-1e24b48120a2, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:14.21261594Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:14.21264454Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:14.233829464Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:14.233857994Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:14.254820745Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:14.254847565Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 185437f9-5949-45df-ad63-9e4a3ae74f66, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:14.276714478Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:14.276747838Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 185437f9-5949-45df-ad63-9e4a3ae74f66, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:14.295994078Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:14.296029108Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:14.315226097Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:14.315261197Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aba8c02d-71d1-44a5-828e-4b07b0810f53, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:14.337632794Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:14.337685626Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aba8c02d-71d1-44a5-828e-4b07b0810f53, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:14.358447195Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:14.358486175Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d3585e5c-c17a-4960-9767-5d4a4b80868c, dataset_name: DatasetName { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:14.379828551Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:14.379862682Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d3585e5c-c17a-4960-9767-5d4a4b80868c, dataset_name: DatasetName { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fab91420-061b-4a94-8963-c1358582fd81, dataset_name: DatasetName { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:14.398355132Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:14.398391002Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fab91420-061b-4a94-8963-c1358582fd81, dataset_name: DatasetName { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6762670-2805-43eb-8b0e-62853b8b03ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:14.420312155Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:14.420354146Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d6762670-2805-43eb-8b0e-62853b8b03ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2024f4e2-927b-4e90-b569-39dcf76e856b, dataset_name: DatasetName { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:14.441831363Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:14.441880553Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2024f4e2-927b-4e90-b569-39dcf76e856b, dataset_name: DatasetName { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:14.460551866Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/cd56c88b-70fb-4721-a4ca-a4de1b256c20/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:14.460593226Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/464bcedf-19e0-4839-a12d-8d477420e762/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:14.460820419Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:14.460959941Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:14.560893575Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:14.560948225Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:14.560954536Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:14.560961406Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:14.560966596Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:14.560972966Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:14.560976596Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:14.560980006Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:14.560985046Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:14.561005726Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:14.561010596Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:14.561014326Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:14.561017826Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:14.561021256Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:14.561026317Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:14.561031669Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:14.561035025Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:14.56103824Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:14.561041203Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:14.561045694Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"af2149be-0adc-4b93-bfea-17a3fdb02732","remote_addr":"[fd00:1122:3344:101::1]:43420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c\", TimeNotSynchronized), (\"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81\", TimeNotSynchronized), (\"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2\", TimeNotSynchronized), (\"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba\", TimeNotSynchronized), (\"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2\", TimeNotSynchronized), (\"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313\", TimeNotSynchronized), (\"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b\", TimeNotSynchronized), (\"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b\", TimeNotSynchronized), (\"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4\", TimeNotSynchronized), (\"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb\", TimeNotSynchronized), (\"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075\", TimeNotSynchronized), (\"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c\", TimeNotSynchronized), (\"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66\", TimeNotSynchronized), (\"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23\", TimeNotSynchronized), (\"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781\", TimeNotSynchronized), (\"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53\", TimeNotSynchronized), (\"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772\", TimeNotSynchronized), (\"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef\", TimeNotSynchronized), (\"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb\", TimeNotSynchronized)]","latency_us":518453,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:14.561211389Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"851.00719ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"af2149be-0adc-4b93-bfea-17a3fdb02732\", \"content-length\": \"124\", \"date\": \"Sat, 12 Oct 2024 03:52:14 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"af2149be-0adc-4b93-bfea-17a3fdb02732\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d4cfaf42-df38-40c6-be81-9454d6a558a4 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 2767466a-ac74-4716-82bc-b59860a90f48 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5d30c2c5-5827-4b34-ab81-abc29f085544 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: ac80f25b-dafd-4922-8e9b-8a6b5ae14799 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9341fb54-e534-4734-8c8a-b2833e94a40a, kind: Service { id: ac80f25b-dafd-4922-8e9b-8a6b5ae14799 }, name: Name(\"ntp-ac80f25b-dafd-4922-8e9b-8a6b5ae14799\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 46])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0d2c64f-0705-4e92-90c6-5bc73310609a (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ce1b852-d629-4995-8bda-2223e090d747 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0 (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3a86650-54d2-47f3-b719-1e24b48120a2 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: bceffd56-91ce-4f4e-bc98-3823ea9f2972, kind: Service { id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2 }, name: Name(\"external-dns-0e7f7381-564c-4f95-8e8e-4ae9e12efae2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 231])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 5b00adda-efae-4057-bbdc-9e4e9e192aeb, kind: Service { id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b }, name: Name(\"external-dns-3ec2aea5-3e4e-4785-83a2-42c04a52438b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 106])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 4c95767b-a705-4932-acf6-7f257e60acef (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: ce434cc7-d07f-4371-8836-7047706943dd, kind: Service { id: 4c95767b-a705-4932-acf6-7f257e60acef }, name: Name(\"nexus-4c95767b-a705-4932-acf6-7f257e60acef\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 44])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d24f88b5-85d4-41b0-9285-f812ad1f4772 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 704fe666-2d42-412a-9eca-2073bdc12aef, kind: Service { id: d24f88b5-85d4-41b0-9285-f812ad1f4772 }, name: Name(\"nexus-d24f88b5-85d4-41b0-9285-f812ad1f4772\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 98])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 055f90e5-f76f-48a0-93ae-4d52d8b374b4 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: bf4b340c-988d-42cb-b55b-9439e5c74f11, kind: Service { id: 055f90e5-f76f-48a0-93ae-4d52d8b374b4 }, name: Name(\"nexus-055f90e5-f76f-48a0-93ae-4d52d8b374b4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 03484d2d-6ade-48bc-b42f-9afe85526313 (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5db8d812-d71c-4866-80e3-62dcf03b04c2 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 11337050-9a37-49b4-b85c-ccf249a5c075 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 73e5d241-c20b-4f21-a945-5c95b2887c2c (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 185437f9-5949-45df-ad63-9e4a3ae74f66 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: aba8c02d-71d1-44a5-828e-4b07b0810f53 (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781 (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External } } } }, OmicronZoneConfig { id: d3585e5c-c17a-4960-9767-5d4a4b80868c (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External } } } }, OmicronZoneConfig { id: fab91420-061b-4a94-8963-c1358582fd81 (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6762670-2805-43eb-8b0e-62853b8b03ba (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2024f4e2-927b-4e90-b569-39dcf76e856b (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:15.41132721Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfaf42-df38-40c6-be81-9454d6a558a4, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:15.411632364Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:15.411642944Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4cfaf42-df38-40c6-be81-9454d6a558a4, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2767466a-ac74-4716-82bc-b59860a90f48, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:15.435845555Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:15.435880426Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2767466a-ac74-4716-82bc-b59860a90f48, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5d30c2c5-5827-4b34-ab81-abc29f085544, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:15.458632073Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:15.45865937Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5d30c2c5-5827-4b34-ab81-abc29f085544, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0d2c64f-0705-4e92-90c6-5bc73310609a, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:15.481049148Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:15.481082559Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0d2c64f-0705-4e92-90c6-5bc73310609a, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ce1b852-d629-4995-8bda-2223e090d747, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:15.502114171Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:15.502138331Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1ce1b852-d629-4995-8bda-2223e090d747, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:15.523133162Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:15.523157132Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:15.544514038Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:15.544539008Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3a86650-54d2-47f3-b719-1e24b48120a2, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:15.566488962Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:15.566516262Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b3a86650-54d2-47f3-b719-1e24b48120a2, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:15.587175289Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:15.58721653Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:15.608847349Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:15.608888169Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:15.631334429Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:15.631372509Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 185437f9-5949-45df-ad63-9e4a3ae74f66, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:15.652015116Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:15.652054167Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 185437f9-5949-45df-ad63-9e4a3ae74f66, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:15.673476644Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:15.673511494Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:15.695236034Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:15.695273005Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aba8c02d-71d1-44a5-828e-4b07b0810f53, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:15.715028321Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:15.715063161Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aba8c02d-71d1-44a5-828e-4b07b0810f53, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:15.736732571Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:15.736794792Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d3585e5c-c17a-4960-9767-5d4a4b80868c, dataset_name: DatasetName { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:15.756821155Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:15.756856241Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d3585e5c-c17a-4960-9767-5d4a4b80868c, dataset_name: DatasetName { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fab91420-061b-4a94-8963-c1358582fd81, dataset_name: DatasetName { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:15.777964814Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:15.778000045Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fab91420-061b-4a94-8963-c1358582fd81, dataset_name: DatasetName { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6762670-2805-43eb-8b0e-62853b8b03ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:15.799505452Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:15.799541103Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d6762670-2805-43eb-8b0e-62853b8b03ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2024f4e2-927b-4e90-b569-39dcf76e856b, dataset_name: DatasetName { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:15.819458621Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:15.819492481Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2024f4e2-927b-4e90-b569-39dcf76e856b, dataset_name: DatasetName { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:15.839145266Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/cd56c88b-70fb-4721-a4ca-a4de1b256c20/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:15.839192186Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/464bcedf-19e0-4839-a12d-8d477420e762/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:15.839375349Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:15.83949752Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:15.940505007Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:15.940567798Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:15.940575868Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:15.940582798Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:15.940586578Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:15.940591589Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:15.940595178Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:15.940598618Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:15.940603858Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:15.940608689Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:15.940613899Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:15.940618809Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:15.940639939Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:15.940643589Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:15.940647119Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:15.940650529Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:15.940655369Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:15.940658689Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:15.940662139Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:15.940665389Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"92069c02-410e-48fe-8a1d-8c110f111fcc","remote_addr":"[fd00:1122:3344:101::1]:43420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c\", TimeNotSynchronized), (\"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef\", TimeNotSynchronized), (\"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772\", TimeNotSynchronized), (\"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2\", TimeNotSynchronized), (\"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66\", TimeNotSynchronized), (\"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23\", TimeNotSynchronized), (\"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81\", TimeNotSynchronized), (\"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba\", TimeNotSynchronized), (\"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53\", TimeNotSynchronized), (\"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2\", TimeNotSynchronized), (\"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b\", TimeNotSynchronized), (\"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4\", TimeNotSynchronized), (\"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075\", TimeNotSynchronized), (\"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781\", TimeNotSynchronized), (\"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b\", TimeNotSynchronized), (\"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313\", TimeNotSynchronized), (\"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c\", TimeNotSynchronized), (\"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb\", TimeNotSynchronized), (\"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb\", TimeNotSynchronized)]","latency_us":530646,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:15.940812081Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"1.2531694s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"92069c02-410e-48fe-8a1d-8c110f111fcc\", \"content-length\": \"124\", \"date\": \"Sat, 12 Oct 2024 03:52:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"92069c02-410e-48fe-8a1d-8c110f111fcc\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d4cfaf42-df38-40c6-be81-9454d6a558a4 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 2767466a-ac74-4716-82bc-b59860a90f48 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5d30c2c5-5827-4b34-ab81-abc29f085544 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: ac80f25b-dafd-4922-8e9b-8a6b5ae14799 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9341fb54-e534-4734-8c8a-b2833e94a40a, kind: Service { id: ac80f25b-dafd-4922-8e9b-8a6b5ae14799 }, name: Name(\"ntp-ac80f25b-dafd-4922-8e9b-8a6b5ae14799\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 46])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0d2c64f-0705-4e92-90c6-5bc73310609a (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ce1b852-d629-4995-8bda-2223e090d747 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0 (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3a86650-54d2-47f3-b719-1e24b48120a2 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: bceffd56-91ce-4f4e-bc98-3823ea9f2972, kind: Service { id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2 }, name: Name(\"external-dns-0e7f7381-564c-4f95-8e8e-4ae9e12efae2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 231])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 5b00adda-efae-4057-bbdc-9e4e9e192aeb, kind: Service { id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b }, name: Name(\"external-dns-3ec2aea5-3e4e-4785-83a2-42c04a52438b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 106])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 4c95767b-a705-4932-acf6-7f257e60acef (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: ce434cc7-d07f-4371-8836-7047706943dd, kind: Service { id: 4c95767b-a705-4932-acf6-7f257e60acef }, name: Name(\"nexus-4c95767b-a705-4932-acf6-7f257e60acef\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 44])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d24f88b5-85d4-41b0-9285-f812ad1f4772 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 704fe666-2d42-412a-9eca-2073bdc12aef, kind: Service { id: d24f88b5-85d4-41b0-9285-f812ad1f4772 }, name: Name(\"nexus-d24f88b5-85d4-41b0-9285-f812ad1f4772\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 98])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 055f90e5-f76f-48a0-93ae-4d52d8b374b4 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: bf4b340c-988d-42cb-b55b-9439e5c74f11, kind: Service { id: 055f90e5-f76f-48a0-93ae-4d52d8b374b4 }, name: Name(\"nexus-055f90e5-f76f-48a0-93ae-4d52d8b374b4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 03484d2d-6ade-48bc-b42f-9afe85526313 (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5db8d812-d71c-4866-80e3-62dcf03b04c2 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 11337050-9a37-49b4-b85c-ccf249a5c075 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 73e5d241-c20b-4f21-a945-5c95b2887c2c (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 185437f9-5949-45df-ad63-9e4a3ae74f66 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: aba8c02d-71d1-44a5-828e-4b07b0810f53 (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781 (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External } } } }, OmicronZoneConfig { id: d3585e5c-c17a-4960-9767-5d4a4b80868c (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External } } } }, OmicronZoneConfig { id: fab91420-061b-4a94-8963-c1358582fd81 (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6762670-2805-43eb-8b0e-62853b8b03ba (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2024f4e2-927b-4e90-b569-39dcf76e856b (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:17.192253028Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfaf42-df38-40c6-be81-9454d6a558a4, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:17.192563552Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:17.192584032Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4cfaf42-df38-40c6-be81-9454d6a558a4, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2767466a-ac74-4716-82bc-b59860a90f48, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:17.215583748Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:17.215629759Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2767466a-ac74-4716-82bc-b59860a90f48, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5d30c2c5-5827-4b34-ab81-abc29f085544, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:17.237389379Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:17.23743724Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5d30c2c5-5827-4b34-ab81-abc29f085544, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0d2c64f-0705-4e92-90c6-5bc73310609a, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:17.259365993Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:17.259408093Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0d2c64f-0705-4e92-90c6-5bc73310609a, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ce1b852-d629-4995-8bda-2223e090d747, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:17.28164131Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:17.281689165Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1ce1b852-d629-4995-8bda-2223e090d747, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:17.301772331Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:17.301816312Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:17.321676199Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:17.321715599Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3a86650-54d2-47f3-b719-1e24b48120a2, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:17.340737996Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:17.340782326Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b3a86650-54d2-47f3-b719-1e24b48120a2, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:17.358453956Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:17.358675339Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:17.377150429Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:17.377189679Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:17.397046547Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:17.397087927Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 185437f9-5949-45df-ad63-9e4a3ae74f66, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:17.416963505Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:17.417014055Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 185437f9-5949-45df-ad63-9e4a3ae74f66, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:17.437374979Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:17.437419279Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:17.457042953Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:17.457082994Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aba8c02d-71d1-44a5-828e-4b07b0810f53, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:17.475566224Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:17.475603335Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aba8c02d-71d1-44a5-828e-4b07b0810f53, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:17.495511762Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:17.495548593Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d3585e5c-c17a-4960-9767-5d4a4b80868c, dataset_name: DatasetName { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:17.514368247Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:17.514421598Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d3585e5c-c17a-4960-9767-5d4a4b80868c, dataset_name: DatasetName { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fab91420-061b-4a94-8963-c1358582fd81, dataset_name: DatasetName { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:17.532666845Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:17.532710535Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fab91420-061b-4a94-8963-c1358582fd81, dataset_name: DatasetName { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6762670-2805-43eb-8b0e-62853b8b03ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:17.554039841Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:17.554079871Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d6762670-2805-43eb-8b0e-62853b8b03ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2024f4e2-927b-4e90-b569-39dcf76e856b, dataset_name: DatasetName { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:17.577207379Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:17.57725833Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2024f4e2-927b-4e90-b569-39dcf76e856b, dataset_name: DatasetName { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:17.598275451Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/cd56c88b-70fb-4721-a4ca-a4de1b256c20/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:17.598322642Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/464bcedf-19e0-4839-a12d-8d477420e762/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:17.598498034Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:17.598626116Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:17.700107729Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:17.7001711Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:17.70017922Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:17.70018675Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:17.70019216Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:17.70019866Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:17.70020406Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:17.70020954Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:17.70021495Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:17.70022024Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:17.700225511Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:17.700229031Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:17.700233971Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:17.700237771Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:17.700242751Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:17.700267201Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:17.700271151Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:17.700274741Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:17.700280031Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:17.700311081Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"8a2e2f90-7c7e-4e24-b8de-c92d91cea78e","remote_addr":"[fd00:1122:3344:101::1]:43420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b\", TimeNotSynchronized), (\"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772\", TimeNotSynchronized), (\"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313\", TimeNotSynchronized), (\"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb\", TimeNotSynchronized), (\"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075\", TimeNotSynchronized), (\"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781\", TimeNotSynchronized), (\"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba\", TimeNotSynchronized), (\"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c\", TimeNotSynchronized), (\"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2\", TimeNotSynchronized), (\"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c\", TimeNotSynchronized), (\"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b\", TimeNotSynchronized), (\"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53\", TimeNotSynchronized), (\"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef\", TimeNotSynchronized), (\"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4\", TimeNotSynchronized), (\"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2\", TimeNotSynchronized), (\"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81\", TimeNotSynchronized), (\"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66\", TimeNotSynchronized), (\"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23\", TimeNotSynchronized), (\"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb\", TimeNotSynchronized)]","latency_us":509313,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:17.700474334Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"1.185399936s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8a2e2f90-7c7e-4e24-b8de-c92d91cea78e\", \"content-length\": \"124\", \"date\": \"Sat, 12 Oct 2024 03:52:17 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8a2e2f90-7c7e-4e24-b8de-c92d91cea78e\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d4cfaf42-df38-40c6-be81-9454d6a558a4 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 2767466a-ac74-4716-82bc-b59860a90f48 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5d30c2c5-5827-4b34-ab81-abc29f085544 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: ac80f25b-dafd-4922-8e9b-8a6b5ae14799 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9341fb54-e534-4734-8c8a-b2833e94a40a, kind: Service { id: ac80f25b-dafd-4922-8e9b-8a6b5ae14799 }, name: Name(\"ntp-ac80f25b-dafd-4922-8e9b-8a6b5ae14799\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 46])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0d2c64f-0705-4e92-90c6-5bc73310609a (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ce1b852-d629-4995-8bda-2223e090d747 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0 (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3a86650-54d2-47f3-b719-1e24b48120a2 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: bceffd56-91ce-4f4e-bc98-3823ea9f2972, kind: Service { id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2 }, name: Name(\"external-dns-0e7f7381-564c-4f95-8e8e-4ae9e12efae2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 231])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 5b00adda-efae-4057-bbdc-9e4e9e192aeb, kind: Service { id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b }, name: Name(\"external-dns-3ec2aea5-3e4e-4785-83a2-42c04a52438b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 106])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 4c95767b-a705-4932-acf6-7f257e60acef (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: ce434cc7-d07f-4371-8836-7047706943dd, kind: Service { id: 4c95767b-a705-4932-acf6-7f257e60acef }, name: Name(\"nexus-4c95767b-a705-4932-acf6-7f257e60acef\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 44])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d24f88b5-85d4-41b0-9285-f812ad1f4772 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 704fe666-2d42-412a-9eca-2073bdc12aef, kind: Service { id: d24f88b5-85d4-41b0-9285-f812ad1f4772 }, name: Name(\"nexus-d24f88b5-85d4-41b0-9285-f812ad1f4772\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 98])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 055f90e5-f76f-48a0-93ae-4d52d8b374b4 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: bf4b340c-988d-42cb-b55b-9439e5c74f11, kind: Service { id: 055f90e5-f76f-48a0-93ae-4d52d8b374b4 }, name: Name(\"nexus-055f90e5-f76f-48a0-93ae-4d52d8b374b4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 03484d2d-6ade-48bc-b42f-9afe85526313 (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5db8d812-d71c-4866-80e3-62dcf03b04c2 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 11337050-9a37-49b4-b85c-ccf249a5c075 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 73e5d241-c20b-4f21-a945-5c95b2887c2c (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 185437f9-5949-45df-ad63-9e4a3ae74f66 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: aba8c02d-71d1-44a5-828e-4b07b0810f53 (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781 (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External } } } }, OmicronZoneConfig { id: d3585e5c-c17a-4960-9767-5d4a4b80868c (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External } } } }, OmicronZoneConfig { id: fab91420-061b-4a94-8963-c1358582fd81 (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6762670-2805-43eb-8b0e-62853b8b03ba (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2024f4e2-927b-4e90-b569-39dcf76e856b (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:18.883522889Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfaf42-df38-40c6-be81-9454d6a558a4, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:18.883817452Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:18.883834613Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4cfaf42-df38-40c6-be81-9454d6a558a4, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2767466a-ac74-4716-82bc-b59860a90f48, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:18.907086122Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:18.907132263Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2767466a-ac74-4716-82bc-b59860a90f48, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5d30c2c5-5827-4b34-ab81-abc29f085544, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:18.927272587Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:18.927314084Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5d30c2c5-5827-4b34-ab81-abc29f085544, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0d2c64f-0705-4e92-90c6-5bc73310609a, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:18.951485945Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:18.951529955Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0d2c64f-0705-4e92-90c6-5bc73310609a, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ce1b852-d629-4995-8bda-2223e090d747, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:18.971243211Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:18.971283941Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1ce1b852-d629-4995-8bda-2223e090d747, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:18.99129061Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:18.991319421Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:19.01457785Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:19.01461214Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3a86650-54d2-47f3-b719-1e24b48120a2, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:19.040069617Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:19.040101228Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b3a86650-54d2-47f3-b719-1e24b48120a2, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:19.059882754Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:19.059911653Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:19.078768749Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:19.07879763Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:19.101762995Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:19.101792921Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 185437f9-5949-45df-ad63-9e4a3ae74f66, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:19.122435903Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:19.122463843Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 185437f9-5949-45df-ad63-9e4a3ae74f66, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:19.14234639Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:19.142380881Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:19.164302474Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:19.164326914Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aba8c02d-71d1-44a5-828e-4b07b0810f53, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:19.183868607Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:19.183892378Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aba8c02d-71d1-44a5-828e-4b07b0810f53, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:19.203064286Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:19.203090357Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d3585e5c-c17a-4960-9767-5d4a4b80868c, dataset_name: DatasetName { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:19.223262218Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:19.223290538Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d3585e5c-c17a-4960-9767-5d4a4b80868c, dataset_name: DatasetName { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fab91420-061b-4a94-8963-c1358582fd81, dataset_name: DatasetName { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:19.245328492Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:19.245356613Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fab91420-061b-4a94-8963-c1358582fd81, dataset_name: DatasetName { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6762670-2805-43eb-8b0e-62853b8b03ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:19.264462261Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:19.264488751Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d6762670-2805-43eb-8b0e-62853b8b03ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2024f4e2-927b-4e90-b569-39dcf76e856b, dataset_name: DatasetName { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:19.283528088Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:19.283556798Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2024f4e2-927b-4e90-b569-39dcf76e856b, dataset_name: DatasetName { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:19.305158327Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/cd56c88b-70fb-4721-a4ca-a4de1b256c20/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:19.305194417Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/464bcedf-19e0-4839-a12d-8d477420e762/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:19.30538437Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:19.305515481Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:19.40504048Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:19.405096591Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:19.405112401Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:19.405119351Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:19.405126061Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:19.405131461Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:19.405138352Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:19.405144742Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:19.405148592Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:19.405155162Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:19.405160062Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:19.405165262Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:19.405170412Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:19.405173892Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:19.405177732Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:19.405184222Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:19.405187792Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:19.405203812Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:19.405207502Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:19.405244793Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"4e02e5b2-2107-4785-b789-6e50cefed2d6","remote_addr":"[fd00:1122:3344:101::1]:43420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66\", TimeNotSynchronized), (\"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c\", TimeNotSynchronized), (\"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b\", TimeNotSynchronized), (\"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef\", TimeNotSynchronized), (\"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4\", TimeNotSynchronized), (\"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb\", TimeNotSynchronized), (\"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781\", TimeNotSynchronized), (\"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba\", TimeNotSynchronized), (\"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2\", TimeNotSynchronized), (\"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53\", TimeNotSynchronized), (\"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b\", TimeNotSynchronized), (\"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb\", TimeNotSynchronized), (\"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772\", TimeNotSynchronized), (\"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c\", TimeNotSynchronized), (\"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2\", TimeNotSynchronized), (\"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23\", TimeNotSynchronized), (\"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075\", TimeNotSynchronized), (\"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81\", TimeNotSynchronized), (\"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313\", TimeNotSynchronized)]","latency_us":523036,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:19.405398025Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"1.363674473s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4e02e5b2-2107-4785-b789-6e50cefed2d6\", \"content-length\": \"124\", \"date\": \"Sat, 12 Oct 2024 03:52:18 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4e02e5b2-2107-4785-b789-6e50cefed2d6\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d4cfaf42-df38-40c6-be81-9454d6a558a4 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 2767466a-ac74-4716-82bc-b59860a90f48 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5d30c2c5-5827-4b34-ab81-abc29f085544 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: ac80f25b-dafd-4922-8e9b-8a6b5ae14799 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9341fb54-e534-4734-8c8a-b2833e94a40a, kind: Service { id: ac80f25b-dafd-4922-8e9b-8a6b5ae14799 }, name: Name(\"ntp-ac80f25b-dafd-4922-8e9b-8a6b5ae14799\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 46])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0d2c64f-0705-4e92-90c6-5bc73310609a (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ce1b852-d629-4995-8bda-2223e090d747 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0 (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3a86650-54d2-47f3-b719-1e24b48120a2 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: bceffd56-91ce-4f4e-bc98-3823ea9f2972, kind: Service { id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2 }, name: Name(\"external-dns-0e7f7381-564c-4f95-8e8e-4ae9e12efae2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 231])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 5b00adda-efae-4057-bbdc-9e4e9e192aeb, kind: Service { id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b }, name: Name(\"external-dns-3ec2aea5-3e4e-4785-83a2-42c04a52438b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 106])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 4c95767b-a705-4932-acf6-7f257e60acef (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: ce434cc7-d07f-4371-8836-7047706943dd, kind: Service { id: 4c95767b-a705-4932-acf6-7f257e60acef }, name: Name(\"nexus-4c95767b-a705-4932-acf6-7f257e60acef\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 44])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d24f88b5-85d4-41b0-9285-f812ad1f4772 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 704fe666-2d42-412a-9eca-2073bdc12aef, kind: Service { id: d24f88b5-85d4-41b0-9285-f812ad1f4772 }, name: Name(\"nexus-d24f88b5-85d4-41b0-9285-f812ad1f4772\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 98])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 055f90e5-f76f-48a0-93ae-4d52d8b374b4 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: bf4b340c-988d-42cb-b55b-9439e5c74f11, kind: Service { id: 055f90e5-f76f-48a0-93ae-4d52d8b374b4 }, name: Name(\"nexus-055f90e5-f76f-48a0-93ae-4d52d8b374b4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 03484d2d-6ade-48bc-b42f-9afe85526313 (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5db8d812-d71c-4866-80e3-62dcf03b04c2 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 11337050-9a37-49b4-b85c-ccf249a5c075 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 73e5d241-c20b-4f21-a945-5c95b2887c2c (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 185437f9-5949-45df-ad63-9e4a3ae74f66 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: aba8c02d-71d1-44a5-828e-4b07b0810f53 (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781 (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External } } } }, OmicronZoneConfig { id: d3585e5c-c17a-4960-9767-5d4a4b80868c (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External } } } }, OmicronZoneConfig { id: fab91420-061b-4a94-8963-c1358582fd81 (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6762670-2805-43eb-8b0e-62853b8b03ba (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2024f4e2-927b-4e90-b569-39dcf76e856b (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:20.766786241Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfaf42-df38-40c6-be81-9454d6a558a4, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:20.767091464Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:20.767103025Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4cfaf42-df38-40c6-be81-9454d6a558a4, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2767466a-ac74-4716-82bc-b59860a90f48, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:20.790228439Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:20.790262883Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2767466a-ac74-4716-82bc-b59860a90f48, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5d30c2c5-5827-4b34-ab81-abc29f085544, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:20.811142903Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:20.811183519Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5d30c2c5-5827-4b34-ab81-abc29f085544, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0d2c64f-0705-4e92-90c6-5bc73310609a, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:20.831657698Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:20.831696409Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0d2c64f-0705-4e92-90c6-5bc73310609a, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ce1b852-d629-4995-8bda-2223e090d747, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:20.854369231Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:20.854403541Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1ce1b852-d629-4995-8bda-2223e090d747, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:20.875142769Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:20.87517871Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:20.896392474Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:20.896429794Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3a86650-54d2-47f3-b719-1e24b48120a2, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:20.916747677Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:20.916782857Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b3a86650-54d2-47f3-b719-1e24b48120a2, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:20.938167044Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:20.938217374Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:20.959451399Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:20.959491179Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:20.980261838Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:20.980308598Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 185437f9-5949-45df-ad63-9e4a3ae74f66, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:21.000060664Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:21.000098825Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 185437f9-5949-45df-ad63-9e4a3ae74f66, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:21.018558704Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:21.018618149Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:21.03992402Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:21.039977301Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aba8c02d-71d1-44a5-828e-4b07b0810f53, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:21.061199355Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:21.061247276Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aba8c02d-71d1-44a5-828e-4b07b0810f53, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:21.082409679Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:21.08245346Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d3585e5c-c17a-4960-9767-5d4a4b80868c, dataset_name: DatasetName { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:21.100896849Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:21.10093523Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d3585e5c-c17a-4960-9767-5d4a4b80868c, dataset_name: DatasetName { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fab91420-061b-4a94-8963-c1358582fd81, dataset_name: DatasetName { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:21.120835968Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:21.120875678Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fab91420-061b-4a94-8963-c1358582fd81, dataset_name: DatasetName { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6762670-2805-43eb-8b0e-62853b8b03ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:21.141012889Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:21.141048579Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d6762670-2805-43eb-8b0e-62853b8b03ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2024f4e2-927b-4e90-b569-39dcf76e856b, dataset_name: DatasetName { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:21.159459548Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:21.159495539Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2024f4e2-927b-4e90-b569-39dcf76e856b, dataset_name: DatasetName { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:21.164729754Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"2.716871143s"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:21.181679945Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/cd56c88b-70fb-4721-a4ca-a4de1b256c20/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:21.181726826Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/464bcedf-19e0-4839-a12d-8d477420e762/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:21.181928698Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:21.18205261Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:21.281464777Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:21.281516548Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:21.281522838Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:21.281536938Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:21.281543868Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:21.281549208Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:21.281555938Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:21.281561218Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:21.281567748Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:21.281573228Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:21.281578518Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:21.281583538Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:21.281590229Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:21.281595369Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:21.281601949Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:21.281607019Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:21.281613669Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:21.281617289Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:21.281633199Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:21.28170146Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"3f98d8ae-d06e-4744-b32e-f85e32dd4a8a","remote_addr":"[fd00:1122:3344:101::1]:43420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313\", TimeNotSynchronized), (\"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2\", TimeNotSynchronized), (\"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb\", TimeNotSynchronized), (\"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781\", TimeNotSynchronized), (\"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4\", TimeNotSynchronized), (\"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075\", TimeNotSynchronized), (\"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66\", TimeNotSynchronized), (\"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c\", TimeNotSynchronized), (\"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23\", TimeNotSynchronized), (\"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b\", TimeNotSynchronized), (\"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81\", TimeNotSynchronized), (\"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2\", TimeNotSynchronized), (\"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb\", TimeNotSynchronized), (\"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c\", TimeNotSynchronized), (\"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b\", TimeNotSynchronized), (\"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef\", TimeNotSynchronized), (\"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772\", TimeNotSynchronized), (\"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba\", TimeNotSynchronized), (\"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53\", TimeNotSynchronized)]","latency_us":516191,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:21.281858792Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"2.197332948s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3f98d8ae-d06e-4744-b32e-f85e32dd4a8a\", \"content-length\": \"124\", \"date\": \"Sat, 12 Oct 2024 03:52:20 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3f98d8ae-d06e-4744-b32e-f85e32dd4a8a\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d4cfaf42-df38-40c6-be81-9454d6a558a4 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 2767466a-ac74-4716-82bc-b59860a90f48 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5d30c2c5-5827-4b34-ab81-abc29f085544 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: ac80f25b-dafd-4922-8e9b-8a6b5ae14799 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9341fb54-e534-4734-8c8a-b2833e94a40a, kind: Service { id: ac80f25b-dafd-4922-8e9b-8a6b5ae14799 }, name: Name(\"ntp-ac80f25b-dafd-4922-8e9b-8a6b5ae14799\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 46])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0d2c64f-0705-4e92-90c6-5bc73310609a (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ce1b852-d629-4995-8bda-2223e090d747 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0 (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3a86650-54d2-47f3-b719-1e24b48120a2 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: bceffd56-91ce-4f4e-bc98-3823ea9f2972, kind: Service { id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2 }, name: Name(\"external-dns-0e7f7381-564c-4f95-8e8e-4ae9e12efae2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 231])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 5b00adda-efae-4057-bbdc-9e4e9e192aeb, kind: Service { id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b }, name: Name(\"external-dns-3ec2aea5-3e4e-4785-83a2-42c04a52438b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 106])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 4c95767b-a705-4932-acf6-7f257e60acef (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: ce434cc7-d07f-4371-8836-7047706943dd, kind: Service { id: 4c95767b-a705-4932-acf6-7f257e60acef }, name: Name(\"nexus-4c95767b-a705-4932-acf6-7f257e60acef\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 44])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d24f88b5-85d4-41b0-9285-f812ad1f4772 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 704fe666-2d42-412a-9eca-2073bdc12aef, kind: Service { id: d24f88b5-85d4-41b0-9285-f812ad1f4772 }, name: Name(\"nexus-d24f88b5-85d4-41b0-9285-f812ad1f4772\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 98])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 055f90e5-f76f-48a0-93ae-4d52d8b374b4 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: bf4b340c-988d-42cb-b55b-9439e5c74f11, kind: Service { id: 055f90e5-f76f-48a0-93ae-4d52d8b374b4 }, name: Name(\"nexus-055f90e5-f76f-48a0-93ae-4d52d8b374b4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 03484d2d-6ade-48bc-b42f-9afe85526313 (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5db8d812-d71c-4866-80e3-62dcf03b04c2 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 11337050-9a37-49b4-b85c-ccf249a5c075 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 73e5d241-c20b-4f21-a945-5c95b2887c2c (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 185437f9-5949-45df-ad63-9e4a3ae74f66 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: aba8c02d-71d1-44a5-828e-4b07b0810f53 (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781 (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External } } } }, OmicronZoneConfig { id: d3585e5c-c17a-4960-9767-5d4a4b80868c (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External } } } }, OmicronZoneConfig { id: fab91420-061b-4a94-8963-c1358582fd81 (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6762670-2805-43eb-8b0e-62853b8b03ba (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2024f4e2-927b-4e90-b569-39dcf76e856b (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:23.474047828Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfaf42-df38-40c6-be81-9454d6a558a4, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:23.474346352Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:23.474354832Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4cfaf42-df38-40c6-be81-9454d6a558a4, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2767466a-ac74-4716-82bc-b59860a90f48, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:23.49832607Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:23.498375451Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2767466a-ac74-4716-82bc-b59860a90f48, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5d30c2c5-5827-4b34-ab81-abc29f085544, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:23.521466888Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:23.521513069Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5d30c2c5-5827-4b34-ab81-abc29f085544, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0d2c64f-0705-4e92-90c6-5bc73310609a, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:23.544485105Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:23.544537425Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0d2c64f-0705-4e92-90c6-5bc73310609a, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ce1b852-d629-4995-8bda-2223e090d747, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:23.565860991Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:23.565910462Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1ce1b852-d629-4995-8bda-2223e090d747, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:23.586528868Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:23.586571739Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:23.607450989Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:23.607491619Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3a86650-54d2-47f3-b719-1e24b48120a2, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:23.627274255Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:23.627312776Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b3a86650-54d2-47f3-b719-1e24b48120a2, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:23.646518187Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:23.646553355Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:23.666949379Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:23.6669838Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:23.686415235Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:23.686446522Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 185437f9-5949-45df-ad63-9e4a3ae74f66, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:23.706641243Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:23.706678104Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 185437f9-5949-45df-ad63-9e4a3ae74f66, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:23.725715811Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:23.725754061Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:23.744815169Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:23.744862659Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aba8c02d-71d1-44a5-828e-4b07b0810f53, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:23.764081398Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:23.764127709Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aba8c02d-71d1-44a5-828e-4b07b0810f53, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:23.784085407Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:23.784124358Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d3585e5c-c17a-4960-9767-5d4a4b80868c, dataset_name: DatasetName { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:23.802571568Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:23.802608498Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d3585e5c-c17a-4960-9767-5d4a4b80868c, dataset_name: DatasetName { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fab91420-061b-4a94-8963-c1358582fd81, dataset_name: DatasetName { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:23.821582024Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:23.821623345Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fab91420-061b-4a94-8963-c1358582fd81, dataset_name: DatasetName { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6762670-2805-43eb-8b0e-62853b8b03ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:23.840627931Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:23.840681752Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d6762670-2805-43eb-8b0e-62853b8b03ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2024f4e2-927b-4e90-b569-39dcf76e856b, dataset_name: DatasetName { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:23.860285306Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:23.860324286Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2024f4e2-927b-4e90-b569-39dcf76e856b, dataset_name: DatasetName { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:23.878634014Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/cd56c88b-70fb-4721-a4ca-a4de1b256c20/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:23.878706215Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/464bcedf-19e0-4839-a12d-8d477420e762/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:23.878956758Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:23.87909238Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:23.979129535Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:23.979193276Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:23.979202306Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:23.979209166Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:23.979217076Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:23.979223326Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:23.979243187Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:23.979250987Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:23.979258677Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:23.979265887Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:23.979273297Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:23.979279197Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:23.979286697Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:23.979294447Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:23.979300337Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:23.979306207Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:23.979310528Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:23.979316517Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:23.979323937Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:23.979380428Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"88df7857-591d-4a18-a737-5fd7186b60e8","remote_addr":"[fd00:1122:3344:101::1]:43420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53\", TimeNotSynchronized), (\"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313\", TimeNotSynchronized), (\"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66\", TimeNotSynchronized), (\"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2\", TimeNotSynchronized), (\"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b\", TimeNotSynchronized), (\"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772\", TimeNotSynchronized), (\"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef\", TimeNotSynchronized), (\"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb\", TimeNotSynchronized), (\"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2\", TimeNotSynchronized), (\"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c\", TimeNotSynchronized), (\"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23\", TimeNotSynchronized), (\"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4\", TimeNotSynchronized), (\"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781\", TimeNotSynchronized), (\"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075\", TimeNotSynchronized), (\"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81\", TimeNotSynchronized), (\"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb\", TimeNotSynchronized), (\"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c\", TimeNotSynchronized), (\"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba\", TimeNotSynchronized), (\"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b\", TimeNotSynchronized)]","latency_us":506581,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:23.9795421Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"948.177705ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"88df7857-591d-4a18-a737-5fd7186b60e8\", \"content-length\": \"124\", \"date\": \"Sat, 12 Oct 2024 03:52:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"88df7857-591d-4a18-a737-5fd7186b60e8\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d4cfaf42-df38-40c6-be81-9454d6a558a4 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 2767466a-ac74-4716-82bc-b59860a90f48 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5d30c2c5-5827-4b34-ab81-abc29f085544 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: ac80f25b-dafd-4922-8e9b-8a6b5ae14799 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9341fb54-e534-4734-8c8a-b2833e94a40a, kind: Service { id: ac80f25b-dafd-4922-8e9b-8a6b5ae14799 }, name: Name(\"ntp-ac80f25b-dafd-4922-8e9b-8a6b5ae14799\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 46])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0d2c64f-0705-4e92-90c6-5bc73310609a (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ce1b852-d629-4995-8bda-2223e090d747 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0 (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3a86650-54d2-47f3-b719-1e24b48120a2 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: bceffd56-91ce-4f4e-bc98-3823ea9f2972, kind: Service { id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2 }, name: Name(\"external-dns-0e7f7381-564c-4f95-8e8e-4ae9e12efae2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 231])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 5b00adda-efae-4057-bbdc-9e4e9e192aeb, kind: Service { id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b }, name: Name(\"external-dns-3ec2aea5-3e4e-4785-83a2-42c04a52438b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 106])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 4c95767b-a705-4932-acf6-7f257e60acef (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: ce434cc7-d07f-4371-8836-7047706943dd, kind: Service { id: 4c95767b-a705-4932-acf6-7f257e60acef }, name: Name(\"nexus-4c95767b-a705-4932-acf6-7f257e60acef\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 44])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d24f88b5-85d4-41b0-9285-f812ad1f4772 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 704fe666-2d42-412a-9eca-2073bdc12aef, kind: Service { id: d24f88b5-85d4-41b0-9285-f812ad1f4772 }, name: Name(\"nexus-d24f88b5-85d4-41b0-9285-f812ad1f4772\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 98])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 055f90e5-f76f-48a0-93ae-4d52d8b374b4 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: bf4b340c-988d-42cb-b55b-9439e5c74f11, kind: Service { id: 055f90e5-f76f-48a0-93ae-4d52d8b374b4 }, name: Name(\"nexus-055f90e5-f76f-48a0-93ae-4d52d8b374b4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 03484d2d-6ade-48bc-b42f-9afe85526313 (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5db8d812-d71c-4866-80e3-62dcf03b04c2 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 11337050-9a37-49b4-b85c-ccf249a5c075 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 73e5d241-c20b-4f21-a945-5c95b2887c2c (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 185437f9-5949-45df-ad63-9e4a3ae74f66 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: aba8c02d-71d1-44a5-828e-4b07b0810f53 (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781 (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External } } } }, OmicronZoneConfig { id: d3585e5c-c17a-4960-9767-5d4a4b80868c (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External } } } }, OmicronZoneConfig { id: fab91420-061b-4a94-8963-c1358582fd81 (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6762670-2805-43eb-8b0e-62853b8b03ba (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2024f4e2-927b-4e90-b569-39dcf76e856b (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:24.926660769Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfaf42-df38-40c6-be81-9454d6a558a4, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:24.927144965Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:24.927158205Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4cfaf42-df38-40c6-be81-9454d6a558a4, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2767466a-ac74-4716-82bc-b59860a90f48, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:24.949616714Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:24.949653985Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2767466a-ac74-4716-82bc-b59860a90f48, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5d30c2c5-5827-4b34-ab81-abc29f085544, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:24.969279459Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:24.96932196Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5d30c2c5-5827-4b34-ab81-abc29f085544, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0d2c64f-0705-4e92-90c6-5bc73310609a, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:24.987714709Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:24.987736039Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0d2c64f-0705-4e92-90c6-5bc73310609a, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ce1b852-d629-4995-8bda-2223e090d747, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:25.005591061Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:25.005626342Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1ce1b852-d629-4995-8bda-2223e090d747, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:25.026839746Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:25.026883396Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:25.047366171Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:25.047401562Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3a86650-54d2-47f3-b719-1e24b48120a2, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:25.066194196Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:25.066230026Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b3a86650-54d2-47f3-b719-1e24b48120a2, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:25.08584603Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:25.085877111Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:25.10506804Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:25.10509911Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:25.123878804Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:25.123914034Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 185437f9-5949-45df-ad63-9e4a3ae74f66, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:25.143956993Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:25.143984244Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 185437f9-5949-45df-ad63-9e4a3ae74f66, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:25.161787966Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:25.161821276Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:25.180318406Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:25.180349186Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aba8c02d-71d1-44a5-828e-4b07b0810f53, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:25.1983084Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:25.198344005Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aba8c02d-71d1-44a5-828e-4b07b0810f53, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:25.21599873Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:25.216030651Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d3585e5c-c17a-4960-9767-5d4a4b80868c, dataset_name: DatasetName { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:25.234294808Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:25.234327458Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d3585e5c-c17a-4960-9767-5d4a4b80868c, dataset_name: DatasetName { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fab91420-061b-4a94-8963-c1358582fd81, dataset_name: DatasetName { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:25.252757338Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:25.252786668Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fab91420-061b-4a94-8963-c1358582fd81, dataset_name: DatasetName { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6762670-2805-43eb-8b0e-62853b8b03ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:25.272419912Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:25.272451993Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d6762670-2805-43eb-8b0e-62853b8b03ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2024f4e2-927b-4e90-b569-39dcf76e856b, dataset_name: DatasetName { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:25.291452189Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:25.29149298Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2024f4e2-927b-4e90-b569-39dcf76e856b, dataset_name: DatasetName { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:25.310323474Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/cd56c88b-70fb-4721-a4ca-a4de1b256c20/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:25.310361955Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/464bcedf-19e0-4839-a12d-8d477420e762/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:25.310559977Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:25.310691239Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:25.407431443Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:25.407488794Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:25.407496934Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:25.407503444Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:25.407510014Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:25.407515394Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:25.407520814Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:25.407527534Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:25.407543684Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:25.407547554Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:25.407552894Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:25.407556534Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:25.407561525Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:25.407566485Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:25.407571265Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:25.407576225Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:25.407581315Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:25.407586125Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:25.407589755Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:25.407635626Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"53541380-2b7e-486f-b12f-b371fdac776d","remote_addr":"[fd00:1122:3344:101::1]:43420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781\", TimeNotSynchronized), (\"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c\", TimeNotSynchronized), (\"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66\", TimeNotSynchronized), (\"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c\", TimeNotSynchronized), (\"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb\", TimeNotSynchronized), (\"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef\", TimeNotSynchronized), (\"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb\", TimeNotSynchronized), (\"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba\", TimeNotSynchronized), (\"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b\", TimeNotSynchronized), (\"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2\", TimeNotSynchronized), (\"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23\", TimeNotSynchronized), (\"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53\", TimeNotSynchronized), (\"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b\", TimeNotSynchronized), (\"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772\", TimeNotSynchronized), (\"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075\", TimeNotSynchronized), (\"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313\", TimeNotSynchronized), (\"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2\", TimeNotSynchronized), (\"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81\", TimeNotSynchronized), (\"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4\", TimeNotSynchronized)]","latency_us":482080,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:25.407795099Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"1.869009429s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"53541380-2b7e-486f-b12f-b371fdac776d\", \"content-length\": \"124\", \"date\": \"Sat, 12 Oct 2024 03:52:25 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"53541380-2b7e-486f-b12f-b371fdac776d\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d4cfaf42-df38-40c6-be81-9454d6a558a4 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 2767466a-ac74-4716-82bc-b59860a90f48 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5d30c2c5-5827-4b34-ab81-abc29f085544 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: ac80f25b-dafd-4922-8e9b-8a6b5ae14799 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9341fb54-e534-4734-8c8a-b2833e94a40a, kind: Service { id: ac80f25b-dafd-4922-8e9b-8a6b5ae14799 }, name: Name(\"ntp-ac80f25b-dafd-4922-8e9b-8a6b5ae14799\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 46])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0d2c64f-0705-4e92-90c6-5bc73310609a (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ce1b852-d629-4995-8bda-2223e090d747 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0 (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3a86650-54d2-47f3-b719-1e24b48120a2 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: bceffd56-91ce-4f4e-bc98-3823ea9f2972, kind: Service { id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2 }, name: Name(\"external-dns-0e7f7381-564c-4f95-8e8e-4ae9e12efae2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 231])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 5b00adda-efae-4057-bbdc-9e4e9e192aeb, kind: Service { id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b }, name: Name(\"external-dns-3ec2aea5-3e4e-4785-83a2-42c04a52438b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 106])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 4c95767b-a705-4932-acf6-7f257e60acef (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: ce434cc7-d07f-4371-8836-7047706943dd, kind: Service { id: 4c95767b-a705-4932-acf6-7f257e60acef }, name: Name(\"nexus-4c95767b-a705-4932-acf6-7f257e60acef\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 44])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d24f88b5-85d4-41b0-9285-f812ad1f4772 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 704fe666-2d42-412a-9eca-2073bdc12aef, kind: Service { id: d24f88b5-85d4-41b0-9285-f812ad1f4772 }, name: Name(\"nexus-d24f88b5-85d4-41b0-9285-f812ad1f4772\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 98])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 055f90e5-f76f-48a0-93ae-4d52d8b374b4 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: bf4b340c-988d-42cb-b55b-9439e5c74f11, kind: Service { id: 055f90e5-f76f-48a0-93ae-4d52d8b374b4 }, name: Name(\"nexus-055f90e5-f76f-48a0-93ae-4d52d8b374b4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 03484d2d-6ade-48bc-b42f-9afe85526313 (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5db8d812-d71c-4866-80e3-62dcf03b04c2 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 11337050-9a37-49b4-b85c-ccf249a5c075 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 73e5d241-c20b-4f21-a945-5c95b2887c2c (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 185437f9-5949-45df-ad63-9e4a3ae74f66 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: aba8c02d-71d1-44a5-828e-4b07b0810f53 (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781 (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External } } } }, OmicronZoneConfig { id: d3585e5c-c17a-4960-9767-5d4a4b80868c (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External } } } }, OmicronZoneConfig { id: fab91420-061b-4a94-8963-c1358582fd81 (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6762670-2805-43eb-8b0e-62853b8b03ba (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2024f4e2-927b-4e90-b569-39dcf76e856b (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:27.272762271Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfaf42-df38-40c6-be81-9454d6a558a4, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:27.273279418Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:27.273286218Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4cfaf42-df38-40c6-be81-9454d6a558a4, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2767466a-ac74-4716-82bc-b59860a90f48, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:27.297274686Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:27.297347197Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2767466a-ac74-4716-82bc-b59860a90f48, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5d30c2c5-5827-4b34-ab81-abc29f085544, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:27.317138993Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:27.317192014Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5d30c2c5-5827-4b34-ab81-abc29f085544, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0d2c64f-0705-4e92-90c6-5bc73310609a, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:27.337417166Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:27.337467816Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0d2c64f-0705-4e92-90c6-5bc73310609a, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ce1b852-d629-4995-8bda-2223e090d747, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:27.355945526Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:27.355994927Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1ce1b852-d629-4995-8bda-2223e090d747, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:27.375438459Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:27.3754959Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:27.393874818Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:27.393929859Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3a86650-54d2-47f3-b719-1e24b48120a2, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:27.414256342Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:27.414310863Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b3a86650-54d2-47f3-b719-1e24b48120a2, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:27.434644906Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:27.434706177Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:27.454714986Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:27.454775767Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:27.474986558Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:27.477350108Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 185437f9-5949-45df-ad63-9e4a3ae74f66, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:27.496239443Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:27.496287453Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 185437f9-5949-45df-ad63-9e4a3ae74f66, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:27.516022919Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:27.516076204Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:27.53460506Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:27.536881058Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aba8c02d-71d1-44a5-828e-4b07b0810f53, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:27.552984426Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:27.55303235Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aba8c02d-71d1-44a5-828e-4b07b0810f53, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:27.571653191Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:27.571710262Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d3585e5c-c17a-4960-9767-5d4a4b80868c, dataset_name: DatasetName { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:27.591223725Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:27.591277396Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d3585e5c-c17a-4960-9767-5d4a4b80868c, dataset_name: DatasetName { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fab91420-061b-4a94-8963-c1358582fd81, dataset_name: DatasetName { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:27.610602096Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:27.610651097Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fab91420-061b-4a94-8963-c1358582fd81, dataset_name: DatasetName { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6762670-2805-43eb-8b0e-62853b8b03ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:27.629576022Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:27.629621673Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d6762670-2805-43eb-8b0e-62853b8b03ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2024f4e2-927b-4e90-b569-39dcf76e856b, dataset_name: DatasetName { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:27.64782765Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:27.64788052Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2024f4e2-927b-4e90-b569-39dcf76e856b, dataset_name: DatasetName { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:27.665780773Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/cd56c88b-70fb-4721-a4ca-a4de1b256c20/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:27.665837264Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/464bcedf-19e0-4839-a12d-8d477420e762/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:27.666037866Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:27.666168358Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:27.761874369Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:27.7619335Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:27.76194212Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:27.76196159Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:27.76196925Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:27.76197616Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:27.76198362Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:27.76199074Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:27.761996531Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:27.762003321Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:27.762009081Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:27.762021511Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:27.762038911Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:27.762044851Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:27.762050131Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:27.762054231Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:27.762059081Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:27.762064732Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:27.762070392Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:27.762081932Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"92aaba65-e929-42c0-ba33-c2533a7cc2df","remote_addr":"[fd00:1122:3344:101::1]:43420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075\", TimeNotSynchronized), (\"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4\", TimeNotSynchronized), (\"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313\", TimeNotSynchronized), (\"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c\", TimeNotSynchronized), (\"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66\", TimeNotSynchronized), (\"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b\", TimeNotSynchronized), (\"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b\", TimeNotSynchronized), (\"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef\", TimeNotSynchronized), (\"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23\", TimeNotSynchronized), (\"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c\", TimeNotSynchronized), (\"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba\", TimeNotSynchronized), (\"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb\", TimeNotSynchronized), (\"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781\", TimeNotSynchronized), (\"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772\", TimeNotSynchronized), (\"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2\", TimeNotSynchronized), (\"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2\", TimeNotSynchronized), (\"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb\", TimeNotSynchronized), (\"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53\", TimeNotSynchronized), (\"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81\", TimeNotSynchronized)]","latency_us":490458,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:27.762265824Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"3.143144496s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"92aaba65-e929-42c0-ba33-c2533a7cc2df\", \"content-length\": \"124\", \"date\": \"Sat, 12 Oct 2024 03:52:27 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"92aaba65-e929-42c0-ba33-c2533a7cc2df\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d4cfaf42-df38-40c6-be81-9454d6a558a4 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 2767466a-ac74-4716-82bc-b59860a90f48 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5d30c2c5-5827-4b34-ab81-abc29f085544 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: ac80f25b-dafd-4922-8e9b-8a6b5ae14799 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9341fb54-e534-4734-8c8a-b2833e94a40a, kind: Service { id: ac80f25b-dafd-4922-8e9b-8a6b5ae14799 }, name: Name(\"ntp-ac80f25b-dafd-4922-8e9b-8a6b5ae14799\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 46])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0d2c64f-0705-4e92-90c6-5bc73310609a (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ce1b852-d629-4995-8bda-2223e090d747 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0 (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3a86650-54d2-47f3-b719-1e24b48120a2 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: bceffd56-91ce-4f4e-bc98-3823ea9f2972, kind: Service { id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2 }, name: Name(\"external-dns-0e7f7381-564c-4f95-8e8e-4ae9e12efae2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 231])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 5b00adda-efae-4057-bbdc-9e4e9e192aeb, kind: Service { id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b }, name: Name(\"external-dns-3ec2aea5-3e4e-4785-83a2-42c04a52438b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 106])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 4c95767b-a705-4932-acf6-7f257e60acef (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: ce434cc7-d07f-4371-8836-7047706943dd, kind: Service { id: 4c95767b-a705-4932-acf6-7f257e60acef }, name: Name(\"nexus-4c95767b-a705-4932-acf6-7f257e60acef\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 44])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d24f88b5-85d4-41b0-9285-f812ad1f4772 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 704fe666-2d42-412a-9eca-2073bdc12aef, kind: Service { id: d24f88b5-85d4-41b0-9285-f812ad1f4772 }, name: Name(\"nexus-d24f88b5-85d4-41b0-9285-f812ad1f4772\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 98])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 055f90e5-f76f-48a0-93ae-4d52d8b374b4 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: bf4b340c-988d-42cb-b55b-9439e5c74f11, kind: Service { id: 055f90e5-f76f-48a0-93ae-4d52d8b374b4 }, name: Name(\"nexus-055f90e5-f76f-48a0-93ae-4d52d8b374b4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 03484d2d-6ade-48bc-b42f-9afe85526313 (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5db8d812-d71c-4866-80e3-62dcf03b04c2 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 11337050-9a37-49b4-b85c-ccf249a5c075 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 73e5d241-c20b-4f21-a945-5c95b2887c2c (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 185437f9-5949-45df-ad63-9e4a3ae74f66 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: aba8c02d-71d1-44a5-828e-4b07b0810f53 (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781 (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External } } } }, OmicronZoneConfig { id: d3585e5c-c17a-4960-9767-5d4a4b80868c (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External } } } }, OmicronZoneConfig { id: fab91420-061b-4a94-8963-c1358582fd81 (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6762670-2805-43eb-8b0e-62853b8b03ba (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2024f4e2-927b-4e90-b569-39dcf76e856b (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:30.89753666Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfaf42-df38-40c6-be81-9454d6a558a4, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:30.897894365Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:30.897905885Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4cfaf42-df38-40c6-be81-9454d6a558a4, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2767466a-ac74-4716-82bc-b59860a90f48, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:30.921149624Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:30.921204675Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2767466a-ac74-4716-82bc-b59860a90f48, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5d30c2c5-5827-4b34-ab81-abc29f085544, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:30.942696182Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:30.942742263Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5d30c2c5-5827-4b34-ab81-abc29f085544, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0d2c64f-0705-4e92-90c6-5bc73310609a, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:30.961139572Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:30.96342361Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0d2c64f-0705-4e92-90c6-5bc73310609a, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ce1b852-d629-4995-8bda-2223e090d747, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:30.981215452Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:30.981272992Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1ce1b852-d629-4995-8bda-2223e090d747, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:30.998391676Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:30.998430706Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:31.019604597Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:31.01966014Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3a86650-54d2-47f3-b719-1e24b48120a2, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:31.03893974Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:31.039004431Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b3a86650-54d2-47f3-b719-1e24b48120a2, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:31.057268248Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:31.057318659Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:31.076353276Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:31.076402916Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:31.094794636Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:31.094841536Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 185437f9-5949-45df-ad63-9e4a3ae74f66, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:31.11284974Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:31.112912231Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 185437f9-5949-45df-ad63-9e4a3ae74f66, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:31.131139328Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:31.131184378Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:31.149215643Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:31.149265783Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aba8c02d-71d1-44a5-828e-4b07b0810f53, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:31.167680183Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:31.167728743Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aba8c02d-71d1-44a5-828e-4b07b0810f53, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:31.186226443Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:31.186268354Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d3585e5c-c17a-4960-9767-5d4a4b80868c, dataset_name: DatasetName { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:31.203992725Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:31.204045905Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d3585e5c-c17a-4960-9767-5d4a4b80868c, dataset_name: DatasetName { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fab91420-061b-4a94-8963-c1358582fd81, dataset_name: DatasetName { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:31.222083616Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:31.22212945Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fab91420-061b-4a94-8963-c1358582fd81, dataset_name: DatasetName { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6762670-2805-43eb-8b0e-62853b8b03ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:31.240122474Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:31.240176365Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d6762670-2805-43eb-8b0e-62853b8b03ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2024f4e2-927b-4e90-b569-39dcf76e856b, dataset_name: DatasetName { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:31.258364472Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:31.258416202Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2024f4e2-927b-4e90-b569-39dcf76e856b, dataset_name: DatasetName { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:31.276322745Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/cd56c88b-70fb-4721-a4ca-a4de1b256c20/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:31.276381286Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/464bcedf-19e0-4839-a12d-8d477420e762/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:31.276576728Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:31.27670502Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:31.369644506Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:31.369706967Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:31.369716097Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:31.369725547Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:31.369734177Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:31.369739607Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:31.369751357Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:31.369757917Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:31.369779398Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:31.369788418Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:31.369796868Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:31.369805418Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:31.369811728Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:31.369821998Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:31.369838878Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:31.369845668Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:31.369852548Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:31.369859019Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:31.369872619Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:31.36993028Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"cb67202d-c59a-4691-8135-cd11c552be31","remote_addr":"[fd00:1122:3344:101::1]:43420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b\", TimeNotSynchronized), (\"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2\", TimeNotSynchronized), (\"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef\", TimeNotSynchronized), (\"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb\", TimeNotSynchronized), (\"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2\", TimeNotSynchronized), (\"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4\", TimeNotSynchronized), (\"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313\", TimeNotSynchronized), (\"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772\", TimeNotSynchronized), (\"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075\", TimeNotSynchronized), (\"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c\", TimeNotSynchronized), (\"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23\", TimeNotSynchronized), (\"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81\", TimeNotSynchronized), (\"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b\", TimeNotSynchronized), (\"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba\", TimeNotSynchronized), (\"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53\", TimeNotSynchronized), (\"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66\", TimeNotSynchronized), (\"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781\", TimeNotSynchronized), (\"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb\", TimeNotSynchronized), (\"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c\", TimeNotSynchronized)]","latency_us":473537,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:31.370097122Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"3.652693851s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"cb67202d-c59a-4691-8135-cd11c552be31\", \"content-length\": \"124\", \"date\": \"Sat, 12 Oct 2024 03:52:30 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"cb67202d-c59a-4691-8135-cd11c552be31\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d4cfaf42-df38-40c6-be81-9454d6a558a4 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 2767466a-ac74-4716-82bc-b59860a90f48 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5d30c2c5-5827-4b34-ab81-abc29f085544 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: ac80f25b-dafd-4922-8e9b-8a6b5ae14799 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9341fb54-e534-4734-8c8a-b2833e94a40a, kind: Service { id: ac80f25b-dafd-4922-8e9b-8a6b5ae14799 }, name: Name(\"ntp-ac80f25b-dafd-4922-8e9b-8a6b5ae14799\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 46])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0d2c64f-0705-4e92-90c6-5bc73310609a (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ce1b852-d629-4995-8bda-2223e090d747 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0 (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3a86650-54d2-47f3-b719-1e24b48120a2 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: bceffd56-91ce-4f4e-bc98-3823ea9f2972, kind: Service { id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2 }, name: Name(\"external-dns-0e7f7381-564c-4f95-8e8e-4ae9e12efae2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 231])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 5b00adda-efae-4057-bbdc-9e4e9e192aeb, kind: Service { id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b }, name: Name(\"external-dns-3ec2aea5-3e4e-4785-83a2-42c04a52438b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 106])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 4c95767b-a705-4932-acf6-7f257e60acef (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: ce434cc7-d07f-4371-8836-7047706943dd, kind: Service { id: 4c95767b-a705-4932-acf6-7f257e60acef }, name: Name(\"nexus-4c95767b-a705-4932-acf6-7f257e60acef\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 44])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d24f88b5-85d4-41b0-9285-f812ad1f4772 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 704fe666-2d42-412a-9eca-2073bdc12aef, kind: Service { id: d24f88b5-85d4-41b0-9285-f812ad1f4772 }, name: Name(\"nexus-d24f88b5-85d4-41b0-9285-f812ad1f4772\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 98])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 055f90e5-f76f-48a0-93ae-4d52d8b374b4 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: bf4b340c-988d-42cb-b55b-9439e5c74f11, kind: Service { id: 055f90e5-f76f-48a0-93ae-4d52d8b374b4 }, name: Name(\"nexus-055f90e5-f76f-48a0-93ae-4d52d8b374b4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 03484d2d-6ade-48bc-b42f-9afe85526313 (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5db8d812-d71c-4866-80e3-62dcf03b04c2 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 11337050-9a37-49b4-b85c-ccf249a5c075 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 73e5d241-c20b-4f21-a945-5c95b2887c2c (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 185437f9-5949-45df-ad63-9e4a3ae74f66 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: aba8c02d-71d1-44a5-828e-4b07b0810f53 (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781 (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External } } } }, OmicronZoneConfig { id: d3585e5c-c17a-4960-9767-5d4a4b80868c (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External } } } }, OmicronZoneConfig { id: fab91420-061b-4a94-8963-c1358582fd81 (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6762670-2805-43eb-8b0e-62853b8b03ba (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2024f4e2-927b-4e90-b569-39dcf76e856b (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:35.01412621Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfaf42-df38-40c6-be81-9454d6a558a4, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:35.014463094Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:35.014470565Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4cfaf42-df38-40c6-be81-9454d6a558a4, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2767466a-ac74-4716-82bc-b59860a90f48, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:35.036232756Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:35.036273716Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2767466a-ac74-4716-82bc-b59860a90f48, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5d30c2c5-5827-4b34-ab81-abc29f085544, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:35.056246815Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:35.056278305Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5d30c2c5-5827-4b34-ab81-abc29f085544, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0d2c64f-0705-4e92-90c6-5bc73310609a, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:35.074085267Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:35.074118797Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0d2c64f-0705-4e92-90c6-5bc73310609a, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ce1b852-d629-4995-8bda-2223e090d747, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:35.095384812Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:35.095423472Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1ce1b852-d629-4995-8bda-2223e090d747, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:35.115451702Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:35.115489792Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:35.135507738Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:35.135539312Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3a86650-54d2-47f3-b719-1e24b48120a2, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:35.154013532Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:35.154058592Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b3a86650-54d2-47f3-b719-1e24b48120a2, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:35.172799076Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:35.172840576Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:35.190533966Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:35.190567757Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:35.209229269Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:35.20926753Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 185437f9-5949-45df-ad63-9e4a3ae74f66, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:35.226792208Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:35.226825538Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 185437f9-5949-45df-ad63-9e4a3ae74f66, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:35.245918606Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:35.245970566Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:35.2639528Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:35.263990351Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aba8c02d-71d1-44a5-828e-4b07b0810f53, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:35.281410027Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:35.281444967Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aba8c02d-71d1-44a5-828e-4b07b0810f53, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:35.300349442Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:35.300389593Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d3585e5c-c17a-4960-9767-5d4a4b80868c, dataset_name: DatasetName { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:35.317981942Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:35.318018349Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d3585e5c-c17a-4960-9767-5d4a4b80868c, dataset_name: DatasetName { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fab91420-061b-4a94-8963-c1358582fd81, dataset_name: DatasetName { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:35.335565601Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:35.335597601Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fab91420-061b-4a94-8963-c1358582fd81, dataset_name: DatasetName { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6762670-2805-43eb-8b0e-62853b8b03ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:35.353447893Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:35.353487054Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d6762670-2805-43eb-8b0e-62853b8b03ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2024f4e2-927b-4e90-b569-39dcf76e856b, dataset_name: DatasetName { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:35.372809144Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:35.372847905Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2024f4e2-927b-4e90-b569-39dcf76e856b, dataset_name: DatasetName { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:35.391424196Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/cd56c88b-70fb-4721-a4ca-a4de1b256c20/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:35.391464417Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/464bcedf-19e0-4839-a12d-8d477420e762/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:35.391632519Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:35.39174174Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:35.487484682Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:35.487561447Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:35.487606013Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:35.487625903Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:35.487634364Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:35.487641344Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:35.487649494Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:35.487656304Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:35.487663894Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:35.487671914Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:35.487684064Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:35.487690894Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:35.487696064Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:35.487702235Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:35.487708744Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:35.487714965Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:35.487721415Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:35.487731805Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:35.487751405Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3292","zone":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:35.487771135Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"a92fa1a4-65ea-4a4e-8993-24426a820005","remote_addr":"[fd00:1122:3344:101::1]:43420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:938","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb\", TimeNotSynchronized), (\"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c\", TimeNotSynchronized), (\"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef\", TimeNotSynchronized), (\"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81\", TimeNotSynchronized), (\"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b\", TimeNotSynchronized), (\"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba\", TimeNotSynchronized), (\"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53\", TimeNotSynchronized), (\"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb\", TimeNotSynchronized), (\"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075\", TimeNotSynchronized), (\"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23\", TimeNotSynchronized), (\"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2\", TimeNotSynchronized), (\"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66\", TimeNotSynchronized), (\"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781\", TimeNotSynchronized), (\"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2\", TimeNotSynchronized), (\"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4\", TimeNotSynchronized), (\"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313\", TimeNotSynchronized), (\"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c\", TimeNotSynchronized), (\"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772\", TimeNotSynchronized), (\"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b\", TimeNotSynchronized)]","latency_us":474798,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:35.487942798Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:495","retry_after":"2.312893442s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a92fa1a4-65ea-4a4e-8993-24426a820005\", \"content-length\": \"124\", \"date\": \"Sat, 12 Oct 2024 03:52:35 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a92fa1a4-65ea-4a4e-8993-24426a820005\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:35.673387466Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:55225"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:35.673481897Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"b98c7729-ad35-4760-a02f-9bd1dead316b","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:55225","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":38,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d4cfaf42-df38-40c6-be81-9454d6a558a4 (service), underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 2767466a-ac74-4716-82bc-b59860a90f48 (service), underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 5d30c2c5-5827-4b34-ab81-abc29f085544 (service), underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: ac80f25b-dafd-4922-8e9b-8a6b5ae14799 (service), underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 9341fb54-e534-4734-8c8a-b2833e94a40a, kind: Service { id: ac80f25b-dafd-4922-8e9b-8a6b5ae14799 }, name: Name(\"ntp-ac80f25b-dafd-4922-8e9b-8a6b5ae14799\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 213, 46])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: b0d2c64f-0705-4e92-90c6-5bc73310609a (service), underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1ce1b852-d629-4995-8bda-2223e090d747 (service), underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0 (service), underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee (service), underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External } } } }, OmicronZoneConfig { id: b3a86650-54d2-47f3-b719-1e24b48120a2 (service), underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2 (service), underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: bceffd56-91ce-4f4e-bc98-3823ea9f2972, kind: Service { id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2 }, name: Name(\"external-dns-0e7f7381-564c-4f95-8e8e-4ae9e12efae2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 231])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b (service), underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 5b00adda-efae-4057-bbdc-9e4e9e192aeb, kind: Service { id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b }, name: Name(\"external-dns-3ec2aea5-3e4e-4785-83a2-42c04a52438b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 106])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 4c95767b-a705-4932-acf6-7f257e60acef (service), underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: ce434cc7-d07f-4371-8836-7047706943dd, kind: Service { id: 4c95767b-a705-4932-acf6-7f257e60acef }, name: Name(\"nexus-4c95767b-a705-4932-acf6-7f257e60acef\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 44])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: d24f88b5-85d4-41b0-9285-f812ad1f4772 (service), underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 704fe666-2d42-412a-9eca-2073bdc12aef, kind: Service { id: d24f88b5-85d4-41b0-9285-f812ad1f4772 }, name: Name(\"nexus-d24f88b5-85d4-41b0-9285-f812ad1f4772\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 98])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 055f90e5-f76f-48a0-93ae-4d52d8b374b4 (service), underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: bf4b340c-988d-42cb-b55b-9439e5c74f11, kind: Service { id: 055f90e5-f76f-48a0-93ae-4d52d8b374b4 }, name: Name(\"nexus-055f90e5-f76f-48a0-93ae-4d52d8b374b4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 03484d2d-6ade-48bc-b42f-9afe85526313 (service), underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb (service), underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5db8d812-d71c-4866-80e3-62dcf03b04c2 (service), underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 11337050-9a37-49b4-b85c-ccf249a5c075 (service), underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 73e5d241-c20b-4f21-a945-5c95b2887c2c (service), underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 185437f9-5949-45df-ad63-9e4a3ae74f66 (service), underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23 (service), underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb (service), underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External } } } }, OmicronZoneConfig { id: aba8c02d-71d1-44a5-828e-4b07b0810f53 (service), underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781 (service), underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External } } } }, OmicronZoneConfig { id: d3585e5c-c17a-4960-9767-5d4a4b80868c (service), underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External } } } }, OmicronZoneConfig { id: fab91420-061b-4a94-8963-c1358582fd81 (service), underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External } } } }, OmicronZoneConfig { id: d6762670-2805-43eb-8b0e-62853b8b03ba (service), underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2024f4e2-927b-4e90-b569-39dcf76e856b (service), underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:37.795566272Z","hostname":"buskin","pid":1572,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:458"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d4cfaf42-df38-40c6-be81-9454d6a558a4, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:37.795858715Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:37.795870786Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d4cfaf42-df38-40c6-be81-9454d6a558a4, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2767466a-ac74-4716-82bc-b59860a90f48, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:37.818936913Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:37.819002263Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2767466a-ac74-4716-82bc-b59860a90f48, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5d30c2c5-5827-4b34-ab81-abc29f085544, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:37.839684981Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:37.839728401Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 5d30c2c5-5827-4b34-ab81-abc29f085544, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b0d2c64f-0705-4e92-90c6-5bc73310609a, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:37.857523093Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:37.859813401Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b0d2c64f-0705-4e92-90c6-5bc73310609a, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1ce1b852-d629-4995-8bda-2223e090d747, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:37.877572492Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:37.877627993Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 1ce1b852-d629-4995-8bda-2223e090d747, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:37.895894671Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:37.895944971Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3c626683-6e71-4b7f-a2b1-dc9974cceae0, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:37.914159548Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:37.914209588Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 4aee4d5a-936f-4d5a-bf11-e3b00a4cc7ee, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b3a86650-54d2-47f3-b719-1e24b48120a2, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:37.931314801Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:37.931379282Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: b3a86650-54d2-47f3-b719-1e24b48120a2, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:37.949822282Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:37.949867392Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:37.968383823Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:37.968429873Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:37.986580859Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:37.98663004Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aed3f8da-f6a1-49dc-b7fa-60c039c84dcb, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 185437f9-5949-45df-ad63-9e4a3ae74f66, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.005441454Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.005490165Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 185437f9-5949-45df-ad63-9e4a3ae74f66, dataset_name: DatasetName { pool_name: ZpoolName { id: bed11bbb-e174-4475-bd7f-0211de5b5302 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.024171627Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.024218398Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0f61af14-0a97-45d6-b6ea-51ac5a92fd23, dataset_name: DatasetName { pool_name: ZpoolName { id: 2b84429f-2c85-4c1f-8f16-c6c336bc6cc2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.042977841Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.043025952Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 0ad3345c-91a2-48a8-b19d-d0a316a1e1bb, dataset_name: DatasetName { pool_name: ZpoolName { id: ddc852e6-47e6-4d51-a45f-adecc30d1fb3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aba8c02d-71d1-44a5-828e-4b07b0810f53, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.061958218Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.062011858Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: aba8c02d-71d1-44a5-828e-4b07b0810f53, dataset_name: DatasetName { pool_name: ZpoolName { id: c0c860af-959d-47ac-8156-a6981a984705 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.081059845Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.081107166Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: a2bf1aec-85b5-4a45-96f5-b2f0e61bd781, dataset_name: DatasetName { pool_name: ZpoolName { id: c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d3585e5c-c17a-4960-9767-5d4a4b80868c, dataset_name: DatasetName { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.099326963Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.099386613Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d3585e5c-c17a-4960-9767-5d4a4b80868c, dataset_name: DatasetName { pool_name: ZpoolName { id: a135191f-1186-46b0-a6c9-d425ec528829 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fab91420-061b-4a94-8963-c1358582fd81, dataset_name: DatasetName { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.118114257Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.118161597Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: fab91420-061b-4a94-8963-c1358582fd81, dataset_name: DatasetName { pool_name: ZpoolName { id: cd705dc8-a605-494a-9c72-bf21f8cc1d40 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d6762670-2805-43eb-8b0e-62853b8b03ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.136011669Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.13605452Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: d6762670-2805-43eb-8b0e-62853b8b03ba, dataset_name: DatasetName { pool_name: ZpoolName { id: 42090923-c06f-4776-b57f-20fba0911cf3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2024f4e2-927b-4e90-b569-39dcf76e856b, dataset_name: DatasetName { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.154732682Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.154782333Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:1059","request":"NewFilesystemRequest { dataset_id: 2024f4e2-927b-4e90-b569-39dcf76e856b, dataset_name: DatasetName { pool_name: ZpoolName { id: f95ba139-c738-48a5-85f9-774bb88e7e5a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.173435015Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/cd56c88b-70fb-4721-a4ca-a4de1b256c20/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.173489626Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/464bcedf-19e0-4839-a12d-8d477420e762/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.173706498Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.1738248Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.262041358Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.273635612Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.320751109Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.321033072Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.321060243Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.32325967Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.323345421Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.323368161Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.345182523Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.345252534Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.500049361Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.500288484Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.500332614Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.501701631Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.540803078Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.540859869Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.540869959Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.540874269Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.540878449Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.541504467Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.547025465Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.551768944Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.556686306Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.561118641Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.567148456Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.572443442Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.710083705Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.715190919Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.719933998Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.724562425Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.729104642Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.734165935Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.738853703Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.743714804Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.748845968Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.754576329Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.760986829Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.767229566Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.772392811Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.778261144Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.78441395Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.78999851Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.796123116Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.802190912Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.809048737Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.817570582Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.825067766Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:38.887543394Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:38.931989637Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"2.480983625s"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.007074809Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.066498122Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.124755256Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.180748474Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.183214594Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.183229335Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.247719987Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte1\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 145, 98]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.247750548Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1266","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.24952984Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.24954372Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.302921505Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte2\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 250, 231]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.302955115Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1266","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.304710747Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.304725537Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.379325945Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte3\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 204, 44]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.379354526Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1266","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.38447865Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.38449531Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.449760512Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte4\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 193, 173]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.449791553Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1266","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.451127009Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.451140279Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.517245262Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte5\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 183, 106]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.517277543Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:1266","zone_type":"external_dns"} {"msg":"no nat entry found for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.518001172Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.518015752Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.518167164Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.518191174Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.518285225Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.518297855Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.518348936Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.518357476Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.518417027Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.518430007Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.518514398Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.519825544Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"creation of nat entry successful for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.519895225Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"creation of nat entry successful for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.52107502Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"creation of nat entry successful for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.523393099Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.523404689Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.523412419Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.573961318Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"creation of nat entry successful for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.574155081Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Configuring new Omicron zone: oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.728270789Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.74707396Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.878734252Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.885254053Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.891001685Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.89787781Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.914155273Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.936099366Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.953126748Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.968771983Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:39.987552656Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:41.557605629Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:41.557966113Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:41.65012837Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:41.650351193Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:41.847287204Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:41.847460517Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:41.862514514Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:41.86299328Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:42.015123213Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:42.015342556Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:43.048976332Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:43.049445837Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:43.517283681Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:43.526207692Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:43.527570699Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:43.528152656Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:43.528314118Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:43.539868552Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:43.549997548Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:43.559519756Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:43.570517623Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:43.570869658Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb\") failed: Property not found. retry in 59.619894ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:44.447771253Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c\") failed: Property not found. retry in 50.953553ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:44.491778231Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81\") failed: Property not found. retry in 55.10318ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:44.534603704Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb\") failed: Property not found. retry in 104.126607ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:44.557899634Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba\") failed: Property not found. retry in 49.642382ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:44.586667052Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2\") failed: Property not found. retry in 57.038106ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:44.657452763Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:44.663320746Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781\") failed: Property not found. retry in 70.355244ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:44.710543684Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c\") failed: Property not found. retry in 71.974403ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:44.746706694Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81\") failed: Property not found. retry in 97.876045ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:47.925682522Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb\") failed: Property not found. retry in 108.714438ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:51.243934774Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba\") failed: Property not found. retry in 64.811765ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:51.690802742Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075\") failed: Property not found. retry in 65.480145ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:51.98353088Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53\") failed: Property not found. retry in 35.897841ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:52.105084133Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b\") failed: Property not found. retry in 50.534862ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:52.483541554Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2\") failed: Property not found. retry in 140.942925ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:52.751007554Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:52.860317194Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781\") failed: Property not found. retry in 94.874486ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:53.088501095Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c\") failed: Property not found. retry in 35.689421ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:53.211795789Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23\") failed: Property not found. retry in 58.864635ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:53.283509255Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c\") failed: Property not found. retry in 151.357313ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:53.425885863Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81\") failed: Property not found. retry in 237.633845ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:53.5204452Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb\") failed: Property not found. retry in 293.542074ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:53.804278703Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba\") failed: Property not found. retry in 237.086647ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:54.061537835Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075\") failed: Property not found. retry in 85.495356ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:54.365689231Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53\") failed: Property not found. retry in 58.743691ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:55.22804983Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b\") failed: Property not found. retry in 56.917402ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:55.516434764Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2\") failed: Property not found. retry in 138.385147ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:55.675992847Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313\") failed: Property not found. retry in 74.974636ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:55.743816329Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781\") failed: Property not found. retry in 160.767789ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:55.887561426Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c\") failed: Property not found. retry in 59.575614ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:55.986511215Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23\") failed: Property not found. retry in 99.627885ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:56.070912163Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c\") failed: Property not found. retry in 209.257176ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:56.803664941Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81\") failed: Property not found. retry in 534.09673ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:57.020295941Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb\") failed: Property not found. retry in 624.594724ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:57.047861393Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba\") failed: Property not found. retry in 498.290109ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:57.172706733Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075\") failed: Property not found. retry in 222.403158ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:57.299152663Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53\") failed: Property not found. retry in 159.456219ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:57.600968971Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b\") failed: Property not found. retry in 262.545669ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:57.741457295Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2\") failed: Property not found. retry in 311.989254ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:57.832641797Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313\") failed: Property not found. retry in 120.121694ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:57.861950871Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781\") failed: Property not found. retry in 560.746809ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:57.892255637Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c\") failed: Property not found. retry in 224.891159ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:57.951598034Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23\") failed: Property not found. retry in 129.368304ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:57.97942838Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c\") failed: Property not found. retry in 790.531053ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:58.041210277Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81\") failed: Property not found. retry in 738.022858ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:58.129863098Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb\") failed: Property not found. retry in 567.243251ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:58.16064321Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba\") failed: Property not found. retry in 903.921864ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:58.189533068Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075\") failed: Property not found. retry in 276.884376ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:58.203384851Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53\") failed: Property not found. retry in 533.844638ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:58.26059493Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b\") failed: Property not found. retry in 438.416225ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:58.338136893Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2\") failed: Property not found. retry in 1.064454057s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:58.366961041Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313\") failed: Property not found. retry in 269.215547ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:58.383920491Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c\") failed: Property not found. retry in 327.689433ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:58.457635997Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23\") failed: Property not found. retry in 301.180598ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:58.485545763Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075\") failed: Property not found. retry in 456.46698ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:58.514154169Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781\") failed: Property not found. retry in 1.090510776s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:58.544983911Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:58.628292746Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"8.634065773s"} {"msg":"wait for service Some(\"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313\") failed: Property not found. retry in 403.137861ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:58.664650357Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb\") failed: Property not found. retry in 649.054346ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:58.738032138Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b\") failed: Property not found. retry in 480.178782ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:58.785408897Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c\") failed: Property not found. retry in 1.130556667s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:58.792180841Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23\") failed: Property not found. retry in 905.723476ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:58.800578965Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53\") failed: Property not found. retry in 601.609202ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:58.807773324Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c\") failed: Property not found. retry in 652.74037ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:58.840495381Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:58.841394932Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:58.841763456Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81\") failed: Property not found. retry in 984.565989ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:58.876734851Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075\") failed: Property not found. retry in 1.315439476s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:58.979698749Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313\") failed: Property not found. retry in 817.139903ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:59.079767032Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66\") failed: Property not found. retry in 50.059557ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:59.087513808Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba\") failed: Property not found. retry in 1.163006625s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:59.101628163Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66\") failed: Property not found. retry in 146.506628ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:59.14732263Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b\") failed: Property not found. retry in 568.720396ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:59.275310659Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66\") failed: Property not found. retry in 105.225783ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:59.303154374Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:59.303961204Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:52:59.304241078Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb\") failed: Property not found. retry in 1.371837203s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:59.398876493Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66\") failed: Property not found. retry in 291.140984ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:59.417625506Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53\") failed: Property not found. retry in 1.256641074s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:59.426414515Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2\") failed: Property not found. retry in 772.246141ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:59.440037034Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2\") failed: Property not found. retry in 62.407481ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:59.469730902Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c\") failed: Property not found. retry in 1.44802752s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:59.50255287Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2\") failed: Property not found. retry in 55.647726ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:59.540983947Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2\") failed: Property not found. retry in 146.128951ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:59.605989794Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781\") failed: Property not found. retry in 843.879206ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:59.644877177Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23\") failed: Property not found. retry in 1.255170499s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:59.716191883Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66\") failed: Property not found. retry in 688.583686ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:59.724725919Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2\") failed: Property not found. retry in 276.319294ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:59.761985061Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b\") failed: Property not found. retry in 772.981148ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:59.854142746Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81\") failed: Property not found. retry in 981.498228ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:59.870475609Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313\") failed: Property not found. retry in 880.369677ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:59.905647315Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c\") failed: Property not found. retry in 1.015858858s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:52:59.932572909Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2\") failed: Property not found. retry in 530.406896ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:00.047923032Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2\") failed: Property not found. retry in 720.660077ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:00.224438673Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba\") failed: Property not found. retry in 519.956857ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:00.278753618Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075\") failed: Property not found. retry in 928.457991ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:00.308684699Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66\") failed: Property not found. retry in 1.46095057s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:00.427168011Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781\") failed: Property not found. retry in 1.459379219s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:00.500787625Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2\") failed: Property not found. retry in 1.456645418s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:00.590563039Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b\") failed: Property not found. retry in 872.385069ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:00.639046561Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53\") failed: Property not found. retry in 1.365937797s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:00.69451773Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb\") failed: Property not found. retry in 966.158436ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:00.782648704Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313\") failed: Property not found. retry in 1.485518844s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:00.796933782Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba\") failed: Property not found. retry in 1.269719642s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:00.811090248Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81\") failed: Property not found. retry in 1.485153008s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:00.862304474Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2\") failed: Property not found. retry in 867.627893ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:00.956194759Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c\") failed: Property not found. retry in 554.280085ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:00.965502375Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c\") failed: Property not found. retry in 657.960034ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:00.976443881Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23\") failed: Property not found. retry in 1.053436258s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:00.987149584Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075\") failed: Property not found. retry in 1.080921369s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:01.250902638Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b\") failed: Property not found. retry in 1.31176751s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:01.526704383Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c\") failed: Property not found. retry in 567.449697ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:01.538201046Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c\") failed: Property not found. retry in 538.809301ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:01.647868818Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:01.709499613Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:01.709556724Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:01.730354572Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:01.730420643Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb\") failed: Property not found. retry in 678.949921ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:01.760646128Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2\") failed: Property not found. retry in 736.198122ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:01.836964135Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66\") failed: Property not found. retry in 1.309181192s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:01.90333119Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781\") failed: Property not found. retry in 546.558553ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:01.977056415Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef\") failed: Property not found. retry in 55.556071ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:01.990898467Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4\") failed: Property not found. retry in 55.050197ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:02.004906311Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23\") failed: Property not found. retry in 943.828888ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:02.056201938Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2\") failed: Property not found. retry in 878.74274ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:02.072344148Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef\") failed: Property not found. retry in 65.382517ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:02.088325427Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4\") failed: Property not found. retry in 82.587487ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:02.103587826Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba\") failed: Property not found. retry in 1.023834799s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:02.118247308Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53\") failed: Property not found. retry in 1.279373197s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:02.131640464Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c\") failed: Property not found. retry in 1.389097844s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:02.14415273Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef\") failed: Property not found. retry in 224.542728ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:02.167106575Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4\") failed: Property not found. retry in 238.940571ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:02.198467154Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c\") failed: Property not found. retry in 637.652715ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:02.210629485Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313\") failed: Property not found. retry in 536.242426ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:02.295722792Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075\") failed: Property not found. retry in 1.078096247s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:02.344811331Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81\") failed: Property not found. retry in 999.113843ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:02.359585425Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef\") failed: Property not found. retry in 213.267788ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:02.406080462Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4\") failed: Property not found. retry in 215.240978ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:02.452421687Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb\") failed: Property not found. retry in 1.084333708s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:02.465419199Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781\") failed: Property not found. retry in 1.008860828s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:02.546590037Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2\") failed: Property not found. retry in 1.118609081s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:02.586826576Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef\") failed: Property not found. retry in 802.258467ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:02.633178572Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4\") failed: Property not found. retry in 1.056237828s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:02.681833356Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:02.734271977Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:02.73452055Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313\") failed: Property not found. retry in 1.479170924s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:02.845422227Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b\") failed: Property not found. retry in 1.054968642s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:02.858190826Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c\") failed: Property not found. retry in 1.212156869s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:02.870726321Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b\") failed: Property not found. retry in 71.945252ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:02.964419385Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2\") failed: Property not found. retry in 1.383338059s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:02.975135028Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23\") failed: Property not found. retry in 1.261064527s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:03.014096111Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b\") failed: Property not found. retry in 147.710555ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:03.047209073Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba\") failed: Property not found. retry in 1.293715009s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:03.1549059Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b\") failed: Property not found. retry in 174.135258ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:03.211350301Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66\") failed: Property not found. retry in 1.448514341s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:03.223591722Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81\") failed: Property not found. retry in 1.407376918s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:03.374952431Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b\") failed: Property not found. retry in 297.277143ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:03.397769135Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53\") failed: Property not found. retry in 927.590907ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:03.423693656Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075\") failed: Property not found. retry in 637.958673ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:03.435360991Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef\") failed: Property not found. retry in 757.698209ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:03.450871264Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c\") failed: Property not found. retry in 1.190374524s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:03.545893914Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb\") failed: Property not found. retry in 771.068925ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:03.563511333Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781\") failed: Property not found. retry in 1.288615692s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:03.577106001Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:03.640177395Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:03.640309396Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b\") failed: Property not found. retry in 812.912047ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:03.708932808Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2\") failed: Property not found. retry in 1.344835632s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:03.721493674Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4\") failed: Property not found. retry in 1.493016758s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:03.751872951Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772\") failed: Property not found. retry in 44.380697ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:03.879659258Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b\") failed: Property not found. retry in 1.353551622s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:03.925841002Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772\") failed: Property not found. retry in 106.304243ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:03.935191468Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772\") failed: Property not found. retry in 184.67187ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:04.054990815Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075\") failed: Property not found. retry in 697.298075ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:04.08601102Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c\") failed: Property not found. retry in 907.686533ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:04.097141668Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef\") failed: Property not found. retry in 625.74699ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:04.221796866Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772\") failed: Property not found. retry in 261.505204ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:04.255899379Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23\") failed: Property not found. retry in 1.039559271s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:04.289240203Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313\") failed: Property not found. retry in 1.385561171s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:04.334651197Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb\") failed: Property not found. retry in 503.596493ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:04.344484629Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53\") failed: Property not found. retry in 585.333538ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:04.361420729Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2\") failed: Property not found. retry in 1.003538607s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:04.372020361Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba\") failed: Property not found. retry in 574.790657ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:04.458278312Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772\") failed: Property not found. retry in 1.145506589s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:04.526095303Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b\") failed: Property not found. retry in 769.085891ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:04.534580239Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66\") failed: Property not found. retry in 1.209221239s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:04.682035539Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c\") failed: Property not found. retry in 955.003744ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:04.746115515Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81\") failed: Property not found. retry in 1.220644139s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:04.864884809Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075\") failed: Property not found. retry in 921.179559ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:04.895129945Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb\") failed: Property not found. retry in 513.82898ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:04.912206707Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef\") failed: Property not found. retry in 906.721027ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:04.929197598Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781\") failed: Property not found. retry in 1.008333177s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:04.939686388Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53\") failed: Property not found. retry in 690.655988ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:04.958495631Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c\") failed: Property not found. retry in 854.259916ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:05.01795646Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba\") failed: Property not found. retry in 680.2744ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:05.044850943Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2\") failed: Property not found. retry in 592.824664ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:05.080899011Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4\") failed: Property not found. retry in 1.324670583s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:05.257621914Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b\") failed: Property not found. retry in 1.334775272s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:05.291319702Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b\") failed: Property not found. retry in 1.41934699s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:05.315111138Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23\") failed: Property not found. retry in 796.897393ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:05.339605252Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2\") failed: Property not found. retry in 617.929685ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:05.385700154Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb\") failed: Property not found. retry in 1.338939574s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:05.435706165Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53\") failed: Property not found. retry in 1.475951803s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:05.659037967Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772\") failed: Property not found. retry in 754.578054ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:05.682236805Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2\") failed: Property not found. retry in 843.266926ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:05.69069978Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c\") failed: Property not found. retry in 714.183173ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:05.709729187Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313\") failed: Property not found. retry in 883.279095ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:05.728848834Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:05.735056071Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:37970"} {"msg":"wait for service Some(\"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba\") failed: Property not found. retry in 1.229618431s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:05.739991042Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:05.762509322Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"febccf6f-90ce-47a0-a0ea-79c6d4ca8c97","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:37970","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":26834,"response_code":"200"} {"msg":"wait for service Some(\"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075\") failed: Property not found. retry in 1.498418528s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:05.825599955Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef\") failed: Property not found. retry in 1.097030643s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:05.844592041Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c\") failed: Property not found. retry in 574.883359ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:05.881350917Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66\") failed: Property not found. retry in 1.351368085s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:05.89926132Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781\") failed: Property not found. retry in 648.810229ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:05.957411691Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2\") failed: Property not found. retry in 568.974842ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:06.014529711Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81\") failed: Property not found. retry in 574.917932ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:06.095964782Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23\") failed: Property not found. retry in 937.486335ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:06.150081134Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:06.253336085Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:06.253386546Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c\") failed: Property not found. retry in 1.184499067s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:06.442098128Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772\") failed: Property not found. retry in 1.045961856s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:06.496505364Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c\") failed: Property not found. retry in 804.18483ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:06.512377831Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb\") failed: Property not found. retry in 35.718196ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:06.521396143Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2\") failed: Property not found. retry in 1.138422455s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:06.547401686Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb\") failed: Property not found. retry in 77.620271ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:06.567940431Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4\") failed: Property not found. retry in 1.215129592s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:06.594272408Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2\") failed: Property not found. retry in 918.268959ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:06.603194248Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781\") failed: Property not found. retry in 684.657591ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:06.615088166Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313\") failed: Property not found. retry in 1.207862126s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:06.623231497Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b\") failed: Property not found. retry in 1.238030553s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:06.637110049Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb\") failed: Property not found. retry in 273.216527ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:06.654241372Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81\") failed: Property not found. retry in 1.198758361s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:06.68063907Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b\") failed: Property not found. retry in 1.162136378s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:06.744883057Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb\") failed: Property not found. retry in 585.843681ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:06.78702229Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb\") failed: Property not found. retry in 347.391028ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:06.938673753Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef\") failed: Property not found. retry in 1.109363368s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:06.951555923Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba\") failed: Property not found. retry in 1.439036992s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:06.97949071Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23\") failed: Property not found. retry in 988.707709ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:07.123133373Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53\") failed: Property not found. retry in 836.492901ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:07.145562672Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66\") failed: Property not found. retry in 1.022432063s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:07.260429167Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb\") failed: Property not found. retry in 1.051987861s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:07.314524568Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781\") failed: Property not found. retry in 1.309624134s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:07.322835051Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c\") failed: Property not found. retry in 764.4848ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:07.330846821Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075\") failed: Property not found. retry in 915.275281ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:07.339508178Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb\") failed: Property not found. retry in 833.717089ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:07.385813753Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2\") failed: Property not found. retry in 1.102896654s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:07.534380427Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772\") failed: Property not found. retry in 1.179636463s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:07.550761551Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c\") failed: Property not found. retry in 729.655854ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:07.635445562Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2\") failed: Property not found. retry in 1.415047817s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:07.697456212Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4\") failed: Property not found. retry in 504.435627ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:07.819627239Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313\") failed: Property not found. retry in 1.133975298s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:07.841892405Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b\") failed: Property not found. retry in 541.892916ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:07.889140732Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81\") failed: Property not found. retry in 729.017307ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:07.901211492Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b\") failed: Property not found. retry in 1.287684764s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:07.919385777Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53\") failed: Property not found. retry in 818.33762ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:07.993766781Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef\") failed: Property not found. retry in 1.191755619s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:08.075177301Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c\") failed: Property not found. retry in 623.61375ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:08.106436339Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23\") failed: Property not found. retry in 1.471273627s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:08.123684054Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb\") failed: Property not found. retry in 598.682381ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:08.231653404Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075\") failed: Property not found. retry in 1.462680269s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:08.265141359Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66\") failed: Property not found. retry in 532.688964ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:08.292416998Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4\") failed: Property not found. retry in 1.417942134s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:08.335528083Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c\") failed: Property not found. retry in 824.079312ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:08.377724897Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb\") failed: Property not found. retry in 1.438401422s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:08.388905416Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba\") failed: Property not found. retry in 536.963681ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:08.432333565Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b\") failed: Property not found. retry in 738.856361ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:08.443398532Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81\") failed: Property not found. retry in 1.128385587s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:08.642708947Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2\") failed: Property not found. retry in 1.491087283s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:08.65344821Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781\") failed: Property not found. retry in 546.328497ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:08.662915638Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c\") failed: Property not found. retry in 792.243856ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:08.741132819Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772\") failed: Property not found. retry in 805.463947ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:08.752936385Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53\") failed: Property not found. retry in 794.737315ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:08.824698166Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66\") failed: Property not found. retry in 874.013909ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:08.839172656Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb\") failed: Property not found. retry in 991.934824ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:08.851097384Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba\") failed: Property not found. retry in 891.051424ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:08.985740475Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313\") failed: Property not found. retry in 1.095664648s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:09.001080636Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2\") failed: Property not found. retry in 578.884944ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:09.127744518Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b\") failed: Property not found. retry in 1.282999494s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:09.197010078Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c\") failed: Property not found. retry in 515.063433ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:09.216928946Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b\") failed: Property not found. retry in 547.492924ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:09.230316512Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781\") failed: Property not found. retry in 1.301141789s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:09.241929376Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef\") failed: Property not found. retry in 515.53451ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:09.279808986Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c\") failed: Property not found. retry in 632.434475ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:09.547225366Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772\") failed: Property not found. retry in 1.471309564s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:09.570701137Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23\") failed: Property not found. retry in 851.893254ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:09.607431083Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53\") failed: Property not found. retry in 1.264850656s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:09.629888502Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2\") failed: Property not found. retry in 542.196178ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:09.719008739Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66\") failed: Property not found. retry in 1.267618861s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:09.734825495Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c\") failed: Property not found. retry in 782.998234ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:09.748713277Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075\") failed: Property not found. retry in 1.118404905s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:09.761405915Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4\") failed: Property not found. retry in 565.347129ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:09.776615134Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81\") failed: Property not found. retry in 1.469954548s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:09.816278656Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef\") failed: Property not found. retry in 1.357429386s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:09.837794263Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb\") failed: Property not found. retry in 1.49476149s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:09.851803677Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b\") failed: Property not found. retry in 765.670707ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:09.863743835Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb\") failed: Property not found. retry in 1.230189168s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:09.87621457Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba\") failed: Property not found. retry in 602.051425ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:09.889197961Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313\") failed: Property not found. retry in 862.665033ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:10.109913441Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2\") failed: Property not found. retry in 944.950817ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:10.173636812Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c\") failed: Property not found. retry in 592.187073ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:10.195880939Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2\") failed: Property not found. retry in 1.230099035s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:10.280166445Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4\") failed: Property not found. retry in 978.177927ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:10.356864357Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23\") failed: Property not found. retry in 1.080842705s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:10.47313189Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b\") failed: Property not found. retry in 1.438789245s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:10.492519561Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba\") failed: Property not found. retry in 1.493432861s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:10.507000621Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c\") failed: Property not found. retry in 775.27214ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:10.54637041Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781\") failed: Property not found. retry in 895.429389ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:10.559162089Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b\") failed: Property not found. retry in 1.358265145s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:10.644310975Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c\") failed: Property not found. retry in 907.619895ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:10.809628448Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075\") failed: Property not found. retry in 1.395217905s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:10.897103024Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53\") failed: Property not found. retry in 927.753365ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:10.910913815Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313\") failed: Property not found. retry in 1.324412277s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:10.9894116Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66\") failed: Property not found. retry in 1.204766254s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:11.01845609Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772\") failed: Property not found. retry in 667.398007ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:11.068092957Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb\") failed: Property not found. retry in 1.455445882s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:11.126159357Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2\") failed: Property not found. retry in 1.448593655s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:11.150989596Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef\") failed: Property not found. retry in 943.337441ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:11.218816397Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81\") failed: Property not found. retry in 745.221031ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:11.298387634Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c\") failed: Property not found. retry in 769.374069ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:11.337178816Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4\") failed: Property not found. retry in 501.58708ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:11.35037157Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb\") failed: Property not found. retry in 1.137862499s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:11.363874358Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781\") failed: Property not found. retry in 1.150324747s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:11.466553992Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2\") failed: Property not found. retry in 1.140477425s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:11.523726952Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23\") failed: Property not found. retry in 1.462726485s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:11.567903351Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c\") failed: Property not found. retry in 1.480034431s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:11.731829846Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772\") failed: Property not found. retry in 1.282160984s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:11.749190181Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53\") failed: Property not found. retry in 1.480636675s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:11.851878746Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4\") failed: Property not found. retry in 788.605046ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:11.864804026Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b\") failed: Property not found. retry in 1.288138488s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:11.94564578Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba\") failed: Property not found. retry in 600.106134ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:12.014369423Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b\") failed: Property not found. retry in 1.311255645s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:12.027758589Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81\") failed: Property not found. retry in 938.128764ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:12.060560156Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c\") failed: Property not found. retry in 739.516498ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:12.121010307Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef\") failed: Property not found. retry in 578.722526ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:12.179130988Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66\") failed: Property not found. retry in 761.821757ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:12.236779423Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075\") failed: Property not found. retry in 1.260427004s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:12.31307038Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313\") failed: Property not found. retry in 719.293479ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:12.326588278Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb\") failed: Property not found. retry in 1.19252125s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:12.518013125Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb\") failed: Property not found. retry in 1.226913972s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:12.600067593Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2\") failed: Property not found. retry in 620.248415ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:12.612363786Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba\") failed: Property not found. retry in 1.497635533s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:12.626856056Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781\") failed: Property not found. retry in 1.226359183s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:12.639851487Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4\") failed: Property not found. retry in 1.188737649s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:12.666086252Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2\") failed: Property not found. retry in 1.331673911s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:12.680264328Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef\") failed: Property not found. retry in 1.037654233s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:12.770747242Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c\") failed: Property not found. retry in 895.829772ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:12.876373143Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81\") failed: Property not found. retry in 1.366658809s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:13.015008994Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66\") failed: Property not found. retry in 605.619858ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:13.029909569Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23\") failed: Property not found. retry in 1.444748997s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:13.048456619Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772\") failed: Property not found. retry in 724.60079ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:13.061518451Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313\") failed: Property not found. retry in 785.945201ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:13.073428719Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c\") failed: Property not found. retry in 890.75892ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:13.227871436Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2\") failed: Property not found. retry in 721.885484ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:13.244517302Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b\") failed: Property not found. retry in 1.249422707s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:13.259598579Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53\") failed: Property not found. retry in 823.977703ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:13.347801824Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b\") failed: Property not found. retry in 907.380795ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:13.365461053Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075\") failed: Property not found. retry in 1.024672225s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:13.590636659Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66\") failed: Property not found. retry in 1.361100487s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:13.652144172Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb\") failed: Property not found. retry in 1.060306591s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:13.728603472Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c\") failed: Property not found. retry in 1.209830078s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:13.788309183Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772\") failed: Property not found. retry in 1.0771806s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:13.802779232Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef\") failed: Property not found. retry in 1.360702473s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:13.829635946Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb\") failed: Property not found. retry in 511.167768ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:13.84690784Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4\") failed: Property not found. retry in 1.218159435s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:13.873777114Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313\") failed: Property not found. retry in 1.359408546s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:13.891118819Z","hostname":"buskin","pid":1572,"zone":"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781\") failed: Property not found. retry in 929.797357ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:13.910201256Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2\") failed: Property not found. retry in 1.39542489s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:13.99032895Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2\") failed: Property not found. retry in 734.374695ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:14.035728174Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c\") failed: Property not found. retry in 1.012469045s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:14.141267324Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba\") failed: Property not found. retry in 974.775524ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:14.163150916Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53\") failed: Property not found. retry in 1.266904062s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:14.192234507Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b\") failed: Property not found. retry in 1.358065837s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:14.294256263Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb\") failed: Property not found. retry in 1.208448646s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:14.376904849Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075\") failed: Property not found. retry in 859.358341ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:14.643134404Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb\") failed: Property not found. retry in 1.380246355s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:14.832452884Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781\") failed: Property not found. retry in 567.757038ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:14.859039065Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772\") failed: Property not found. retry in 1.120303514s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:14.895434316Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c\") failed: Property not found. retry in 579.765119ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:15.023264803Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66\") failed: Property not found. retry in 915.107808ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:15.047321272Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4\") failed: Property not found. retry in 545.542186ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:15.121129048Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba\") failed: Property not found. retry in 637.443184ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:15.167599345Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef\") failed: Property not found. retry in 1.344855351s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:15.262224329Z","hostname":"buskin","pid":1572,"zone":"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781\") failed: Property not found. retry in 1.13087044s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:15.460125696Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b\") failed: Property not found. retry in 1.359160463s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:15.753480708Z","hostname":"buskin","pid":1572,"zone":"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba\") failed: Property not found. retry in 636.322418ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:15.828900954Z","hostname":"buskin","pid":1572,"zone":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:16.191948321Z","hostname":"buskin","pid":1572,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"18.71070937s"} {"msg":"wait for service Some(\"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb\") failed: Property not found. retry in 841.149879ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:16.237496106Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb\") failed: Property not found. retry in 699.957183ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:17.107701049Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb\") failed: Property not found. retry in 1.456037972s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:17.827647906Z","hostname":"buskin","pid":1572,"zone":"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:19.317625961Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_nexus_055f90e5-f76f-48a0-93ae-4d52d8b374b4"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:19.317688642Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_external_dns_3ec2aea5-3e4e-4785-83a2-42c04a52438b"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:19.317706512Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_crucible_pantry_73e5d241-c20b-4f21-a945-5c95b2887c2c"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:19.317713452Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_crucible_0f61af14-0a97-45d6-b6ea-51ac5a92fd23"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:19.317722753Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_crucible_d3585e5c-c17a-4960-9767-5d4a4b80868c"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:19.317734653Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_crucible_fab91420-061b-4a94-8963-c1358582fd81"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:19.317741363Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_crucible_0ad3345c-91a2-48a8-b19d-d0a316a1e1bb"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:19.317748583Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_nexus_d24f88b5-85d4-41b0-9285-f812ad1f4772"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:19.317772583Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_crucible_d6762670-2805-43eb-8b0e-62853b8b03ba"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:19.317806683Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_crucible_pantry_11337050-9a37-49b4-b85c-ccf249a5c075"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:19.317812834Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_crucible_aba8c02d-71d1-44a5-828e-4b07b0810f53"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:19.317817723Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_crucible_2024f4e2-927b-4e90-b569-39dcf76e856b"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:19.317826124Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_external_dns_0e7f7381-564c-4f95-8e8e-4ae9e12efae2"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:19.317834864Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_nexus_4c95767b-a705-4932-acf6-7f257e60acef"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:19.317841234Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_clickhouse_aed3f8da-f6a1-49dc-b7fa-60c039c84dcb"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:19.317846464Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_crucible_pantry_5db8d812-d71c-4866-80e3-62dcf03b04c2"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:19.317854954Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_oximeter_03484d2d-6ade-48bc-b42f-9afe85526313"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:19.317860364Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_crucible_a2bf1aec-85b5-4a45-96f5-b2f0e61bd781"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:19.317877554Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:3288","zone":"oxz_crucible_185437f9-5949-45df-ad63-9e4a3ae74f66"} {"msg":"Writing ledger to /pool/int/cd56c88b-70fb-4721-a4ca-a4de1b256c20/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:19.31993056Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/464bcedf-19e0-4839-a12d-8d477420e762/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:19.323526934Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:19.327298931Z","hostname":"buskin","pid":1572,"uri":"/omicron-zones","method":"PUT","req_id":"5f5cd28a-6315-4e71-a882-dea38d68752b","remote_addr":"[fd00:1122:3344:101::1]:43420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":41580765,"response_code":"204"} {"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:19.337776331Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1322"} {"msg":"Writing ledger to /pool/int/cd56c88b-70fb-4721-a4ca-a4de1b256c20/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:19.338062135Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/464bcedf-19e0-4839-a12d-8d477420e762/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:19.338247627Z","hostname":"buskin","pid":1572,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:19.345599519Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:707"} {"msg":"Nexus address: [fd00:1122:3344:101::c]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:19.347157418Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:720"} {"msg":"rack_network_config: RackNetworkConfigV2 {\n bfd: [],\n bgp: [],\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV2 {\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.1.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n autoneg: false,\n bgp_peers: [],\n lldp: None,\n port: \"qsfp0\",\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.1.100,\n rib_priority: None,\n vlan_id: None,\n },\n ],\n switch: Switch0,\n uplink_port_fec: None,\n uplink_port_speed: Speed40G,\n },\n ],\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:19.381756317Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/2b4f7d90-e216-4aab-97d4-74d0f80bf858/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:19.384450851Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/2b4f7d90-e216-4aab-97d4-74d0f80bf858/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:19.51976469Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/2b4f7d90-e216-4aab-97d4-74d0f80bf858/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:19.674227608Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/2b4f7d90-e216-4aab-97d4-74d0f80bf858/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:19.858935151Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/2b4f7d90-e216-4aab-97d4-74d0f80bf858/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:20.111401135Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/2b4f7d90-e216-4aab-97d4-74d0f80bf858/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:20.254892246Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/2b4f7d90-e216-4aab-97d4-74d0f80bf858/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:20.448726952Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/2b4f7d90-e216-4aab-97d4-74d0f80bf858/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:20.778176031Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/2b4f7d90-e216-4aab-97d4-74d0f80bf858/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:20.988593063Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/2b4f7d90-e216-4aab-97d4-74d0f80bf858/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:21.260145973Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/2b4f7d90-e216-4aab-97d4-74d0f80bf858/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:21.926189992Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/2b4f7d90-e216-4aab-97d4-74d0f80bf858/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:22.727925074Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/2b4f7d90-e216-4aab-97d4-74d0f80bf858/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:23.70727774Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/2b4f7d90-e216-4aab-97d4-74d0f80bf858/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:24.457221039Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/2b4f7d90-e216-4aab-97d4-74d0f80bf858/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:25.129660757Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/b8a45863-ed9f-4b08-b97f-6fcba7a5a294)","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:25.3756385Z","hostname":"buskin","pid":1572,"component":"NexusNotifierTask","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/nexus.rs:527"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/b8a45863-ed9f-4b08-b97f-6fcba7a5a294)","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:53:26.250692452Z","hostname":"buskin","pid":1572,"component":"NexusNotifierTask","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/nexus.rs:527"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/b8a45863-ed9f-4b08-b97f-6fcba7a5a294)","v":0,"name":"SledAgent","level":50,"time":"2024-10-12T03:53:26.494713291Z","hostname":"buskin","pid":1572,"component":"ProbeManager","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:230"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/2b4f7d90-e216-4aab-97d4-74d0f80bf858/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:26.66221738Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"66ad1fb5-78b3-421f-8af3-86179c781315\", \"content-length\": \"173\", \"date\": \"Sat, 12 Oct 2024 03:53:30 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"b8a45863-ed9f-4b08-b97f-6fcba7a5a294\\\"\", request_id: \"66ad1fb5-78b3-421f-8af3-86179c781315\" }","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:30.139937492Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:949"} {"msg":"Successfully put SledAgentInfo to nexus","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:30.323591951Z","hostname":"buskin","pid":1572,"component":"NexusNotifierTask","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/nexus.rs:500","new_generation":"1","old_generation":"1"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:30.507091799Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:44911"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:30.509190715Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:30.509393978Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/cd56c88b-70fb-4721-a4ca-a4de1b256c20/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:30.509764382Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/464bcedf-19e0-4839-a12d-8d477420e762/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:30.511618275Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:30.704069094Z","hostname":"buskin","pid":1572,"uri":"/inventory","method":"GET","req_id":"eb4e087a-8e3d-403c-b220-90f4c2a68bd7","remote_addr":"[fd00:1122:3344:101::c]:44911","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":196090,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:32.10146444Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:52093"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:32.103360664Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 5b00adda-efae-4057-bbdc-9e4e9e192aeb, kind: Service { id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b }, name: Name(\"external-dns-3ec2aea5-3e4e-4785-83a2-42c04a52438b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 106])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: bceffd56-91ce-4f4e-bc98-3823ea9f2972, kind: Service { id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2 }, name: Name(\"external-dns-0e7f7381-564c-4f95-8e8e-4ae9e12efae2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 231])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 704fe666-2d42-412a-9eca-2073bdc12aef, kind: Service { id: d24f88b5-85d4-41b0-9285-f812ad1f4772 }, name: Name(\"nexus-d24f88b5-85d4-41b0-9285-f812ad1f4772\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 98])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: bf4b340c-988d-42cb-b55b-9439e5c74f11, kind: Service { id: 055f90e5-f76f-48a0-93ae-4d52d8b374b4 }, name: Name(\"nexus-055f90e5-f76f-48a0-93ae-4d52d8b374b4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ce434cc7-d07f-4371-8836-7047706943dd, kind: Service { id: 4c95767b-a705-4932-acf6-7f257e60acef }, name: Name(\"nexus-4c95767b-a705-4932-acf6-7f257e60acef\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 44])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:32.103775389Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:32.103798799Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:32.10380567Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:32.10382194Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:32.10383802Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:32.10384378Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:32.10387Z","hostname":"buskin","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"0848ce5b-ce0e-4b1b-b0d0-83f41801ef4b","remote_addr":"[fd00:1122:3344:101::c]:52093","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1937,"response_code":"204"} {"msg":"Handoff to Nexus is complete","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:32.105385659Z","hostname":"buskin","pid":1572,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:959"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:32.365979164Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:37314"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:32.367029277Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:32.367039577Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Reading ledger from /pool/int/cd56c88b-70fb-4721-a4ca-a4de1b256c20/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:32.367071917Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/464bcedf-19e0-4839-a12d-8d477420e762/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:32.367488392Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:32.544985956Z","hostname":"buskin","pid":1572,"uri":"/inventory","method":"GET","req_id":"50e596c0-a674-452e-98f6-fa1d5560939f","remote_addr":"[fd00:1122:3344:101::c]:37314","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":178571,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:33.436768516Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:49433"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:33.437019759Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 5b00adda-efae-4057-bbdc-9e4e9e192aeb, kind: Service { id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b }, name: Name(\"external-dns-3ec2aea5-3e4e-4785-83a2-42c04a52438b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 106])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: bceffd56-91ce-4f4e-bc98-3823ea9f2972, kind: Service { id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2 }, name: Name(\"external-dns-0e7f7381-564c-4f95-8e8e-4ae9e12efae2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 231])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 704fe666-2d42-412a-9eca-2073bdc12aef, kind: Service { id: d24f88b5-85d4-41b0-9285-f812ad1f4772 }, name: Name(\"nexus-d24f88b5-85d4-41b0-9285-f812ad1f4772\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 98])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: bf4b340c-988d-42cb-b55b-9439e5c74f11, kind: Service { id: 055f90e5-f76f-48a0-93ae-4d52d8b374b4 }, name: Name(\"nexus-055f90e5-f76f-48a0-93ae-4d52d8b374b4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ce434cc7-d07f-4371-8836-7047706943dd, kind: Service { id: 4c95767b-a705-4932-acf6-7f257e60acef }, name: Name(\"nexus-4c95767b-a705-4932-acf6-7f257e60acef\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 44])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:33.43707711Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:33.437115761Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:33.437131991Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:33.437138001Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:33.437151451Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:33.437165201Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:33.437185071Z","hostname":"buskin","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"371e4432-ea9d-44f0-8512-3359bdaeb2d0","remote_addr":"[fd00:1122:3344:101::b]:49433","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":329,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:33.805697346Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:49748"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:33.805931269Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 5b00adda-efae-4057-bbdc-9e4e9e192aeb, kind: Service { id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b }, name: Name(\"external-dns-3ec2aea5-3e4e-4785-83a2-42c04a52438b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 106])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: bceffd56-91ce-4f4e-bc98-3823ea9f2972, kind: Service { id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2 }, name: Name(\"external-dns-0e7f7381-564c-4f95-8e8e-4ae9e12efae2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 231])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 704fe666-2d42-412a-9eca-2073bdc12aef, kind: Service { id: d24f88b5-85d4-41b0-9285-f812ad1f4772 }, name: Name(\"nexus-d24f88b5-85d4-41b0-9285-f812ad1f4772\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 98])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: bf4b340c-988d-42cb-b55b-9439e5c74f11, kind: Service { id: 055f90e5-f76f-48a0-93ae-4d52d8b374b4 }, name: Name(\"nexus-055f90e5-f76f-48a0-93ae-4d52d8b374b4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ce434cc7-d07f-4371-8836-7047706943dd, kind: Service { id: 4c95767b-a705-4932-acf6-7f257e60acef }, name: Name(\"nexus-4c95767b-a705-4932-acf6-7f257e60acef\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 44])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:33.80598134Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:33.80601341Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:33.80602314Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:33.80602808Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:33.80603344Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:33.80604365Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:33.80606661Z","hostname":"buskin","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"435f9ba0-45ff-4c3f-a45d-63e981831fa9","remote_addr":"[fd00:1122:3344:101::c]:49748","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":292,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:33.88177915Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:55938"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:33.881975543Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 5b00adda-efae-4057-bbdc-9e4e9e192aeb, kind: Service { id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b }, name: Name(\"external-dns-3ec2aea5-3e4e-4785-83a2-42c04a52438b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 106])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: bceffd56-91ce-4f4e-bc98-3823ea9f2972, kind: Service { id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2 }, name: Name(\"external-dns-0e7f7381-564c-4f95-8e8e-4ae9e12efae2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 231])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 704fe666-2d42-412a-9eca-2073bdc12aef, kind: Service { id: d24f88b5-85d4-41b0-9285-f812ad1f4772 }, name: Name(\"nexus-d24f88b5-85d4-41b0-9285-f812ad1f4772\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 98])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: bf4b340c-988d-42cb-b55b-9439e5c74f11, kind: Service { id: 055f90e5-f76f-48a0-93ae-4d52d8b374b4 }, name: Name(\"nexus-055f90e5-f76f-48a0-93ae-4d52d8b374b4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ce434cc7-d07f-4371-8836-7047706943dd, kind: Service { id: 4c95767b-a705-4932-acf6-7f257e60acef }, name: Name(\"nexus-4c95767b-a705-4932-acf6-7f257e60acef\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 44])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:33.882021424Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:33.882048154Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:33.882054874Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:33.882063444Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:33.882085084Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:33.882092094Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:33.882100554Z","hostname":"buskin","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"fc41a927-4809-4cfa-a397-2cf8521d41b0","remote_addr":"[fd00:1122:3344:101::a]:55938","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":238,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:35.802743936Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:53610"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:35.803674357Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"5dee387c-94c9-4312-b51b-5aa0275cb06b","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:53610","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":36,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:40.942568178Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:45397"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:40.944141057Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"fa4e8d13-12e6-4df0-b0be-c6cf35781e3e","remote_addr":"[fd00:1122:3344:101::c]:45397","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1511,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:40.993146085Z","hostname":"buskin","pid":1572,"sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:40.993305547Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"737a65b8-d376-4505-9d72-7730bae65f41","remote_addr":"[fd00:1122:3344:101::c]:45397","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1187,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:41.116949642Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"fb1f99ee-3c6a-461f-8134-a65ba7a21541","remote_addr":"[fd00:1122:3344:101::c]:45397","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":2780,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:59.225533391Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:50184"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:59.227409974Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"5f7a497d-7bd8-4e0c-8800-140fc3e545bb","remote_addr":"[fd00:1122:3344:101::c]:50184","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1196,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:59.240559927Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:53068"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:59.24073365Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"bb2f3be0-39f1-411e-8664-8badaa198eea","remote_addr":"[fd00:1122:3344:101::a]:53068","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":30,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:59.276113289Z","hostname":"buskin","pid":1572,"sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:59.276134999Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"0d967d98-f5f9-4c5f-9d8f-134d71b4734c","remote_addr":"[fd00:1122:3344:101::c]:50184","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1062,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:59.289715238Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"5df15d8e-a0e9-4b42-9fed-5bf45e15e587","remote_addr":"[fd00:1122:3344:101::c]:50184","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":327,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:59.30037257Z","hostname":"buskin","pid":1572,"sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:59.30040183Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"a3f12534-f0f8-4433-b81f-1d8959881a80","remote_addr":"[fd00:1122:3344:101::a]:53068","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1053,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:59.310371584Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"de4559cd-5993-44dc-9eb7-37a95eb134c5","remote_addr":"[fd00:1122:3344:101::a]:53068","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":171,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:59.377328475Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:65035"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:59.377471347Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"a0de8a9c-c23a-410b-b4a1-b1d5185445e8","remote_addr":"[fd00:1122:3344:101::b]:65035","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":39,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:59.395860455Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:39027"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:59.396968999Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"009bd11b-6fba-4673-bc8d-68edd730f010","remote_addr":"[fd00:1122:3344:101::c]:39027","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1051,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:59.408594513Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:33039"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:59.40915038Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:59.492491905Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:59.557175268Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"42a3baf2-81e0-4cdf-b636-30874279ea38","remote_addr":"[fd00:1122:3344:101::c]:33039","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":148535,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:59.557421771Z","hostname":"buskin","pid":1572,"sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:59.557452351Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:48809"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:59.557460161Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"b6efd261-f6ce-406d-b854-ce75af5ee677","remote_addr":"[fd00:1122:3344:101::b]:65035","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":132,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:59.557531862Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:52414"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:59.557601993Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:34583"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:59.557854026Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"41e8faa4-8c3b-43d3-a15e-5eb213c9a40b","remote_addr":"[fd00:1122:3344:101::a]:48809","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":224,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:59.558527644Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"5d9e3366-3ce2-49cb-98ea-07c16150b5b7","remote_addr":"[fd00:1122:3344:101::c]:33039","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1028,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:59.558726077Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:59.563178782Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"1a752094-df41-4be7-a548-b57d6d9bf41e","remote_addr":"[fd00:1122:3344:101::b]:34583","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":4741,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:59.569603472Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"45b9b389-943f-4a1e-9c0a-7a9c45caf7d0","remote_addr":"[fd00:1122:3344:101::b]:65035","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":318,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:59.576386296Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:51170"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:59.603348111Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"Attempting to update network config with generation: 2, current_generation: 1","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:59.613472727Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/cd56c88b-70fb-4721-a4ca-a4de1b256c20/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:59.613639269Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/464bcedf-19e0-4839-a12d-8d477420e762/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:59.615576063Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 2","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:59.616294342Z","hostname":"buskin","pid":1572,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:59.616780568Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"PUT","req_id":"11ae60bc-e54e-4c53-be8d-2c3a7d74452a","remote_addr":"[fd00:1122:3344:101::c]:33039","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":4486,"response_code":"204"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:59.666719488Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:59.666806859Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"79789894-81fb-463b-9402-5a258cda223b","remote_addr":"[fd00:1122:3344:101::a]:52414","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":108949,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:59.670183151Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"682d48fe-7b9b-402a-b58d-d4a744d4eca5","remote_addr":"[fd00:1122:3344:101::a]:52414","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":417,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:59.700585048Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:59.750960103Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"1d9c60af-c60e-440b-8099-e9f5bf422f13","remote_addr":"[fd00:1122:3344:101::b]:51170","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":174390,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:53:59.751375429Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"8887d01e-fa67-4ca8-957e-f07451a47436","remote_addr":"[fd00:1122:3344:101::b]:51170","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":177,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:05.010145167Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:34475","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","component":"producer-server","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::d]:59149"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:05.062460666Z","hostname":"buskin","pid":1572,"uri":"/b8a45863-ed9f-4b08-b97f-6fcba7a5a294","method":"GET","req_id":"8f30b414-500e-4196-9bba-0757b4475f2f","remote_addr":"[fd00:1122:3344:101::d]:59149","local_addr":"[fd00:1122:3344:101::1]:34475","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","component":"producer-server","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":51230,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:05.856492573Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:49005"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:05.856579724Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"ee2543f7-b573-4c8a-8d43-6374b92b7ab0","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:49005","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":32,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:29.201382444Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:47215"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:29.201664928Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"21aed160-6b31-4a55-9b4a-22834a3175d7","remote_addr":"[fd00:1122:3344:101::a]:47215","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":110,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:29.245621212Z","hostname":"buskin","pid":1572,"sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:29.245663193Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"1520af12-9068-42d0-abca-5268139eaefa","remote_addr":"[fd00:1122:3344:101::a]:47215","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":126,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:29.263769587Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"a0b1e9ed-c41b-4c80-961b-1c248326111d","remote_addr":"[fd00:1122:3344:101::a]:47215","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1312,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:29.381757622Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:49201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:29.381858233Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"136e6213-11c2-4d81-9e58-6722c8231644","remote_addr":"[fd00:1122:3344:101::c]:49201","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":56,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:29.406188635Z","hostname":"buskin","pid":1572,"sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:29.406206966Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"ac3a6f35-a8d9-4858-b969-3483f0908b31","remote_addr":"[fd00:1122:3344:101::c]:49201","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":59,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:29.417619697Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"b40c96a3-09dd-46fa-aaf5-84bf4cc5262a","remote_addr":"[fd00:1122:3344:101::c]:49201","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":189,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:29.469759545Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:38468"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:29.469880456Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:29.509211625Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:29.57891052Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"256ab128-b62c-4246-be56-3cebdf97debb","remote_addr":"[fd00:1122:3344:101::a]:38468","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":109067,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:29.579069432Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:48749"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:29.579112082Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:55917"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:29.579155663Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:36604"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:29.579400006Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:29.588066993Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:37876"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:29.588233265Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"d2f59777-f1b8-4ad9-930b-220e2d611aee","remote_addr":"[fd00:1122:3344:101::c]:48749","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8886,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:29.588292226Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"95117bda-1f70-4ee7-b898-f21b2a0fa217","remote_addr":"[fd00:1122:3344:101::b]:55917","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8884,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:29.588422448Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"999abd37-2072-495e-8e77-524bff058c40","remote_addr":"[fd00:1122:3344:101::a]:38468","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":9075,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:29.58864088Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"6cffe60f-ee3f-405e-be88-41d6bf698434","remote_addr":"[fd00:1122:3344:101::a]:37876","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":315,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:29.630184676Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:29.640554635Z","hostname":"buskin","pid":1572,"sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:29.640607776Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"927cbee8-9cc9-4f07-8f2a-bbe9beffd1b5","remote_addr":"[fd00:1122:3344:101::b]:55917","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":169,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:29.641298074Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:46437"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:29.692338408Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:29.692413159Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"7061e6ae-dddf-49b5-809e-2b9931da5e83","remote_addr":"[fd00:1122:3344:101::c]:36604","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":113105,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:29.696148355Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"cb26e596-00d3-496a-87d4-4f289d52764e","remote_addr":"[fd00:1122:3344:101::c]:36604","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":553,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:29.707775849Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:63758"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:29.708167674Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"d6707d9d-b56a-427f-aa8b-1dd4c1b13122","remote_addr":"[fd00:1122:3344:101::b]:63758","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":240,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:29.715404054Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"b2c7ff09-75f3-4d52-b5f7-98affb9cea94","remote_addr":"[fd00:1122:3344:101::b]:55917","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":274,"response_code":"204"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:29.728940602Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:29.783926475Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"d0f87855-1bdf-40e3-8697-4f615190336c","remote_addr":"[fd00:1122:3344:101::b]:46437","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":142031,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:29.784262159Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"3980d4d1-b8bf-4c6e-9602-9ab9e54b2b55","remote_addr":"[fd00:1122:3344:101::b]:46437","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":150,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:35.016928845Z","hostname":"buskin","pid":1572,"uri":"/b8a45863-ed9f-4b08-b97f-6fcba7a5a294","method":"GET","req_id":"072df100-021e-4bc7-aac4-528b00a975e0","remote_addr":"[fd00:1122:3344:101::d]:59149","local_addr":"[fd00:1122:3344:101::1]:34475","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","component":"producer-server","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":7296,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:35.889124881Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:62664"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:35.889239683Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"7497c751-5477-4b59-aebd-084ce9dbefa5","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:62664","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":38,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:59.160780992Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:58133"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:59.161818335Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"322dffb4-8f3f-4c6d-ac29-f8ffe6ad6b98","remote_addr":"[fd00:1122:3344:101::c]:58133","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":959,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:59.181726762Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:56266"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:59.181841044Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"a9d616b4-7ef1-40d8-871c-98aa43765c91","remote_addr":"[fd00:1122:3344:101::a]:56266","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":49,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:59.261236579Z","hostname":"buskin","pid":1572,"sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:59.26127011Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"90ecacef-ad2a-4f8b-a4b3-c1112ab40731","remote_addr":"[fd00:1122:3344:101::a]:56266","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":80,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:59.267146212Z","hostname":"buskin","pid":1572,"sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:59.267176853Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"e4c37e6d-4bc5-4a89-85a0-1c34aa57e186","remote_addr":"[fd00:1122:3344:101::c]:58133","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":933,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:59.274267541Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"167fa6cb-3402-4fd7-a950-350a70492068","remote_addr":"[fd00:1122:3344:101::a]:56266","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":207,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:59.278818617Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"a84a7fba-1955-43c6-bf45-0dbead7aedf5","remote_addr":"[fd00:1122:3344:101::c]:58133","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":154,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:59.317270145Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:38119"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:59.317384886Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:59.362889131Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:59.424239362Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"d2c82127-518b-4831-b966-c22bb4379e45","remote_addr":"[fd00:1122:3344:101::c]:38119","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":106905,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:59.424418155Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:62462"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:59.424452105Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:41233"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:59.424740489Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:59.42481771Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"e3b8d0c7-f284-440a-889b-d3885c3fd8c6","remote_addr":"[fd00:1122:3344:101::c]:38119","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":283,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:59.428669628Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"75a0fe13-8947-4923-9a42-1fff79050ee8","remote_addr":"[fd00:1122:3344:101::b]:41233","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":3854,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:59.455254727Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:34235"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:59.459163186Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"29c14a20-8c6f-48dc-bacd-7465c6883bec","remote_addr":"[fd00:1122:3344:101::a]:34235","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":321,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:59.466322525Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:59.477331101Z","hostname":"buskin","pid":1572,"sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:59.477392172Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"e1e55279-653e-43f9-a21c-e057e8e27deb","remote_addr":"[fd00:1122:3344:101::b]:41233","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":160,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:59.481586534Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:52163"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:59.482609217Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"4910fe89-6a79-406d-ac1c-e0bc39562fc4","remote_addr":"[fd00:1122:3344:101::c]:52163","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":866,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:59.485680405Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:57553"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:59.485871148Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"a1f66e7b-9de8-4a11-8a3b-2c9119e17505","remote_addr":"[fd00:1122:3344:101::b]:57553","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":98,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:59.490391124Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"d99b3f56-81c3-45c1-849b-f2dea789de21","remote_addr":"[fd00:1122:3344:101::b]:41233","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":260,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:59.496349608Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:46944"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:59.526174368Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"55de293e-51ab-4a45-bca0-ac89061b328f","remote_addr":"[fd00:1122:3344:101::a]:62462","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":101558,"response_code":"204"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:59.526258509Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:59.530179298Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"e907a86f-edf8-4766-9f9a-68f84e2332b4","remote_addr":"[fd00:1122:3344:101::a]:62462","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":790,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:59.564615585Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:59.620655921Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"b74d838f-af15-4c77-8eba-598b3088866b","remote_addr":"[fd00:1122:3344:101::b]:46944","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":124193,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:54:59.621055536Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"a736344a-9954-4012-b128-f88d26e8d68d","remote_addr":"[fd00:1122:3344:101::b]:46944","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":174,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:05.017613946Z","hostname":"buskin","pid":1572,"uri":"/b8a45863-ed9f-4b08-b97f-6fcba7a5a294","method":"GET","req_id":"8585b2d4-b0a9-45e6-b112-41edb075adc7","remote_addr":"[fd00:1122:3344:101::d]:59149","local_addr":"[fd00:1122:3344:101::1]:34475","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","component":"producer-server","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":7389,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:05.922164024Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:50346"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:05.922309886Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"161a48e1-d79d-4807-8aa5-4ee137d68212","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:50346","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":52,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:29.367936287Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:54559"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:29.369331475Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"8cb535c5-7afb-4d64-8b30-49e58c39daf9","remote_addr":"[fd00:1122:3344:101::a]:54559","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1329,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:29.411725021Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:35674"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:29.411867933Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"ba11ae4d-8093-4073-a737-f080a72dfb5c","remote_addr":"[fd00:1122:3344:101::b]:35674","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":77,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:29.412238617Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:44919"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:29.413014477Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"65cae6e4-83d0-4747-94a5-d57e2b16ff40","remote_addr":"[fd00:1122:3344:101::c]:44919","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":29,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:29.436412168Z","hostname":"buskin","pid":1572,"sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:29.436475168Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"501cd3d2-1b6c-4fe8-a0f2-698ccadcab9e","remote_addr":"[fd00:1122:3344:101::a]:54559","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":177,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:29.44868825Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"3a9a725a-1ad4-4cd1-b1aa-47ec90b5932d","remote_addr":"[fd00:1122:3344:101::a]:54559","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":205,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:29.484223081Z","hostname":"buskin","pid":1572,"sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:29.484257982Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"e60e139b-9d3e-43ff-9bc0-6c56f3eee25c","remote_addr":"[fd00:1122:3344:101::c]:44919","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":78,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:29.490475709Z","hostname":"buskin","pid":1572,"sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:29.490489169Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"adf576f8-fde6-4f07-9fe8-a0641a91e8ca","remote_addr":"[fd00:1122:3344:101::b]:35674","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":41,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:29.49623671Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"53777e82-552b-4890-b5cb-657f341e89f3","remote_addr":"[fd00:1122:3344:101::c]:44919","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":2591,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:29.502366216Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"495f7ad1-fda5-4023-86ac-6eff95d66bd9","remote_addr":"[fd00:1122:3344:101::b]:35674","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":181,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:29.508622034Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:32798"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:29.508723375Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:29.55338282Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:29.613113661Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"fc0f8665-4696-4deb-bbb6-ae7cf205e6a6","remote_addr":"[fd00:1122:3344:101::a]:32798","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":104428,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:29.613263453Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:36191"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:29.613300503Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:52904"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:29.613349014Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:58161"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:29.613393434Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:51833"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:29.613664838Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:29.61790031Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"c5be337b-98da-4154-9f37-40f285752176","remote_addr":"[fd00:1122:3344:101::a]:32798","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":4175,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:29.618024172Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"1f887886-0e5b-456d-8cca-11560e2dec47","remote_addr":"[fd00:1122:3344:101::c]:51833","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":4223,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:29.618080103Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"76a948fc-8bc3-480b-9744-e8e08f6ba123","remote_addr":"[fd00:1122:3344:101::a]:58161","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":4296,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:29.618153444Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"3601095f-4c18-4253-a9ed-a6a89d3b8a76","remote_addr":"[fd00:1122:3344:101::b]:52904","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":4484,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:29.649746006Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:29.703928088Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"472838e7-6438-47dc-be00-0ca8b4281a25","remote_addr":"[fd00:1122:3344:101::c]:36191","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":90397,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:29.704619557Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"fb6804d7-bac0-4ef1-97c7-5e1554e97499","remote_addr":"[fd00:1122:3344:101::c]:36191","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":135,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:29.729913131Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:45370"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:29.730024112Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:29.764672543Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:29.819338471Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"515a2b2d-c72d-423c-9234-c9b96351e46b","remote_addr":"[fd00:1122:3344:101::b]:45370","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":89361,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:29.819573724Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"d229daba-55e5-4360-a13a-c38100ead068","remote_addr":"[fd00:1122:3344:101::b]:45370","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":107,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:35.020958681Z","hostname":"buskin","pid":1572,"uri":"/b8a45863-ed9f-4b08-b97f-6fcba7a5a294","method":"GET","req_id":"36aee992-0a66-4e64-9602-a79935383977","remote_addr":"[fd00:1122:3344:101::d]:59149","local_addr":"[fd00:1122:3344:101::1]:34475","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","component":"producer-server","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":10110,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:35.956723727Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:56514"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:35.956808958Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"3edf63d0-8f28-4f4b-9ac3-7e9d9c748cd9","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:56514","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":30,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:59.219271996Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:59720"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:59.220652814Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"a2b0ef4f-73e9-4392-bf3b-778a6889a89d","remote_addr":"[fd00:1122:3344:101::c]:59720","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1287,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:59.235614019Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:48030"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:59.235947493Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"697dea20-b68d-4d75-89fd-7c59bd45f1ad","remote_addr":"[fd00:1122:3344:101::a]:48030","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":168,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:59.257543881Z","hostname":"buskin","pid":1572,"sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:59.257582812Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"5c985983-f591-4258-ae16-1927d7d62045","remote_addr":"[fd00:1122:3344:101::a]:48030","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":92,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:59.277917944Z","hostname":"buskin","pid":1572,"sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:59.277945475Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"49fe0d24-5db5-4c83-b94d-312810b0ac5c","remote_addr":"[fd00:1122:3344:101::c]:59720","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":88,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:59.292111581Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"cf3f0541-d6e0-466b-b112-4b792580dd66","remote_addr":"[fd00:1122:3344:101::c]:59720","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":295,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:59.329801978Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:42209"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:59.329980761Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"4fba6167-69f4-479a-8944-8d36f49b1e4a","remote_addr":"[fd00:1122:3344:101::a]:42209","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":142,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:59.333727327Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"cdb9a379-c765-4f73-b36c-8186dd6966b0","remote_addr":"[fd00:1122:3344:101::a]:48030","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":175,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:59.365527192Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:63782"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:59.365686574Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"f1644606-6a71-46c7-b84e-43f2d45b5722","remote_addr":"[fd00:1122:3344:101::c]:63782","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":95,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:59.388279874Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:52185"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:59.388380466Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:59.428426293Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:59.486269961Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"309a6ea2-4eea-4eb5-84cd-02057488354a","remote_addr":"[fd00:1122:3344:101::c]:52185","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":97929,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:59.486506354Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:33609"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:59.486567884Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:61116"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:59.486627355Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:37019"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:59.486813458Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"24b82e0f-d16c-40e8-a6bb-721ddbff4c0a","remote_addr":"[fd00:1122:3344:101::c]:52185","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":231,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:59.48700466Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"eb9507ec-dc1b-4172-8b44-b3119c0d055c","remote_addr":"[fd00:1122:3344:101::b]:61116","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":165,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:59.4870231Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:59.487082391Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"ca526be9-e916-4553-a8ea-f3cd8d83cac2","remote_addr":"[fd00:1122:3344:101::b]:37019","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":307,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:59.506965688Z","hostname":"buskin","pid":1572,"sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:59.507046519Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"fdbd032a-68c9-4f04-907e-5de522593e6d","remote_addr":"[fd00:1122:3344:101::b]:61116","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":723,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:59.507421373Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:58204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:59.519513933Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"0cd95315-6488-45e5-abd1-6ecfb27ae3eb","remote_addr":"[fd00:1122:3344:101::b]:61116","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":942,"response_code":"204"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:59.528493335Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:59.582385954Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:59.582449675Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"9ea00f17-4be4-46cc-a26c-c5bb7a19087f","remote_addr":"[fd00:1122:3344:101::a]:33609","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":95599,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:59.587377826Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"e770e69b-e370-479f-9c4a-a078e37b3a98","remote_addr":"[fd00:1122:3344:101::a]:33609","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":659,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:59.621203196Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:59.678662289Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"50fb69c8-9824-4b38-b075-5baa66674cc7","remote_addr":"[fd00:1122:3344:101::b]:58204","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":171169,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:59.679085234Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"7dec018e-4943-49f3-b292-e00fc6ba106c","remote_addr":"[fd00:1122:3344:101::b]:58204","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":161,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:55:59.737493602Z","hostname":"buskin","pid":1572,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:05.02533843Z","hostname":"buskin","pid":1572,"uri":"/b8a45863-ed9f-4b08-b97f-6fcba7a5a294","method":"GET","req_id":"d628f93c-142b-42cf-8982-c1e1202c08e2","remote_addr":"[fd00:1122:3344:101::d]:59149","local_addr":"[fd00:1122:3344:101::1]:34475","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","component":"producer-server","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":6664,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:05.99122103Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:47391"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:05.991323211Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"f20d336e-da25-4809-9d01-89dcdb95ffd5","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:47391","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":40,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:29.298001607Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:52495"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:29.299724588Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"c070388b-bc40-4bf2-83df-3ae2685972a9","remote_addr":"[fd00:1122:3344:101::c]:52495","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1636,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:29.38279235Z","hostname":"buskin","pid":1572,"sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:29.38282644Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"cd875c03-a056-4674-81ae-af2ac6cd3c38","remote_addr":"[fd00:1122:3344:101::c]:52495","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":79,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:29.394843069Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"cb101cb4-28f2-46dc-b27b-be9994885741","remote_addr":"[fd00:1122:3344:101::c]:52495","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":183,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:29.433135415Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:40521"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:29.433264846Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"25abcf05-ca77-4f99-b814-73088642d4fb","remote_addr":"[fd00:1122:3344:101::c]:40521","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":100,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:29.45614728Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:58790"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:29.456249801Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:29.498554856Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:29.567949868Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"f13de4c0-88f5-4e1f-91e2-739f0ffdf5fe","remote_addr":"[fd00:1122:3344:101::c]:58790","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":111741,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:29.56812956Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:61246"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:29.568182451Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:50042"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:29.568241332Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:39257"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:29.568480604Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"c2bba599-74f9-4c54-b0a9-ba329b7807f9","remote_addr":"[fd00:1122:3344:101::b]:39257","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":130,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:29.568590646Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"ae8d3c8d-de61-47a3-9274-765723aa1da8","remote_addr":"[fd00:1122:3344:101::a]:61246","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":150,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:29.568640226Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"4c486d53-afc3-4413-9b98-1428c08a5eca","remote_addr":"[fd00:1122:3344:101::c]:58790","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":339,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:29.569594338Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"b419c4c1-dfe5-4792-b973-8d3d46d863e0","remote_addr":"[fd00:1122:3344:101::b]:50042","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1118,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:29.620160226Z","hostname":"buskin","pid":1572,"sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:29.620192077Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"42389405-a933-4e30-a766-85dc07679145","remote_addr":"[fd00:1122:3344:101::b]:39257","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":109,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:29.630245181Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"36d77d8c-ae71-43fe-9699-6abac52494ba","remote_addr":"[fd00:1122:3344:101::b]:39257","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":308,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:29.642141839Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:39412"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:29.642938939Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"67acef5b-3a5e-4e93-968e-85b1f5497e4d","remote_addr":"[fd00:1122:3344:101::a]:39412","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":745,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:29.657567441Z","hostname":"buskin","pid":1572,"sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:29.657592621Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"4f47c081-5d23-42c6-9f66-b77901f50b16","remote_addr":"[fd00:1122:3344:101::a]:61246","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":67,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:29.66317465Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:41810"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:29.663306762Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:29.700027197Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:29.755082861Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"cbdd6956-2ed1-488c-a1cb-3c357e52c191","remote_addr":"[fd00:1122:3344:101::a]:41810","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":91850,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:29.755641668Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"4e0cfdac-09d7-4e2f-97b6-fb7f434dbedb","remote_addr":"[fd00:1122:3344:101::a]:41810","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":295,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:29.755687159Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"3c275038-4477-40d4-8a38-1334f226712b","remote_addr":"[fd00:1122:3344:101::a]:61246","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":364,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:29.768530518Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:39321"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:29.768756501Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:29.809268034Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:29.863247284Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"13249647-fa9e-4a15-a72a-26dc1a3ac5bd","remote_addr":"[fd00:1122:3344:101::b]:39321","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":94629,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:29.863788161Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"feb0d340-457c-4902-926a-4d95e57f4d09","remote_addr":"[fd00:1122:3344:101::b]:39321","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":254,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:35.095576524Z","hostname":"buskin","pid":1572,"uri":"/b8a45863-ed9f-4b08-b97f-6fcba7a5a294","method":"GET","req_id":"35ddc547-eac7-4dfe-8761-4cee21325b2f","remote_addr":"[fd00:1122:3344:101::d]:59149","local_addr":"[fd00:1122:3344:101::1]:34475","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","component":"producer-server","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":7382,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:36.025865282Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:41155"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:36.026625522Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"4e460c3a-4651-46b6-8c0f-0218cbf7354f","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:41155","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":712,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:59.129134785Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:61295"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:59.130297089Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"c64b1be5-2167-49d2-ae9d-79dc083d96bb","remote_addr":"[fd00:1122:3344:101::c]:61295","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1093,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:59.172854548Z","hostname":"buskin","pid":1572,"sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:59.172886838Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"7b6f2ad5-7561-439a-a99b-05e4f1534808","remote_addr":"[fd00:1122:3344:101::c]:61295","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1056,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:59.197807458Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"125eeaf7-9e48-4282-be83-f82a717f5838","remote_addr":"[fd00:1122:3344:101::c]:61295","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":244,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:59.209769516Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:63956"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:59.211078622Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"06b0571c-91c3-4814-be60-39db27eb7c99","remote_addr":"[fd00:1122:3344:101::a]:63956","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1239,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:59.272996452Z","hostname":"buskin","pid":1572,"sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:59.273028582Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"00f0053e-8f51-4954-b484-8bd33fb452bb","remote_addr":"[fd00:1122:3344:101::a]:63956","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1004,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:59.285486806Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"e7370af3-8773-4694-b937-8e193cdca1e6","remote_addr":"[fd00:1122:3344:101::a]:63956","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":211,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:59.315420978Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:63867"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:59.31558066Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"3559f7a1-091d-4f32-b795-93e907c68385","remote_addr":"[fd00:1122:3344:101::c]:63867","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":129,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:59.316288248Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:41156"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:59.317307221Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:59.358990108Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:59.416487992Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"f7d0dd20-3f54-432a-a9c9-af59622aaed2","remote_addr":"[fd00:1122:3344:101::a]:41156","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":100122,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:59.416736245Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:56305"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:59.416790796Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:38364"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:59.416845207Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:53747"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:59.417062289Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:59.41709889Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"ba6f002e-1dac-498e-a0e2-a04dc40a3d31","remote_addr":"[fd00:1122:3344:101::a]:41156","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":278,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:59.41713489Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"854de36d-300d-498e-8ce9-d341a8b74d70","remote_addr":"[fd00:1122:3344:101::b]:38364","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":159,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:59.422315655Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:61059"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:59.422520437Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"f4ddc711-cb8f-4400-8240-c3d349ed5b98","remote_addr":"[fd00:1122:3344:101::a]:53747","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":5441,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:59.422821991Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"d0d56871-aae0-4854-a6a9-bd4190d9e11c","remote_addr":"[fd00:1122:3344:101::b]:61059","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":258,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:59.458311781Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:59.464348496Z","hostname":"buskin","pid":1572,"sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:59.464402727Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"7224a64e-e68f-4b1d-9631-9950a0661e1e","remote_addr":"[fd00:1122:3344:101::b]:38364","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":180,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:59.474875587Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"cd6f69a4-8336-4603-8b82-57168a0e2918","remote_addr":"[fd00:1122:3344:101::b]:38364","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":970,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:59.479002688Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:34633"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:59.522032583Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:59.522104933Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"fd32e8dc-f8cd-440c-8166-f9709ebab00c","remote_addr":"[fd00:1122:3344:101::c]:56305","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":105166,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:59.52584608Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"d5eba868-fc12-4e3f-a925-265925702d1b","remote_addr":"[fd00:1122:3344:101::c]:56305","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1066,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:59.5548467Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:59.611387942Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"0436e5c9-3046-40ca-b54d-70c4113d5e4a","remote_addr":"[fd00:1122:3344:101::b]:34633","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":132302,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:56:59.611733976Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"8277fe67-8b45-4827-8239-050dba913967","remote_addr":"[fd00:1122:3344:101::b]:34633","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":150,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:05.096056855Z","hostname":"buskin","pid":1572,"uri":"/b8a45863-ed9f-4b08-b97f-6fcba7a5a294","method":"GET","req_id":"a71c1ddf-acc3-415d-abb9-3f587f75f6bc","remote_addr":"[fd00:1122:3344:101::d]:59149","local_addr":"[fd00:1122:3344:101::1]:34475","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","component":"producer-server","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":7486,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:06.060951362Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:56446"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:06.061046243Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"b867ef10-3676-48de-ab44-f50fda9c7562","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:56446","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":34,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:29.354804767Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:64984"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:29.354980449Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"ac3fb6d9-7fc9-4d72-ab7e-0fe829033d20","remote_addr":"[fd00:1122:3344:101::a]:64984","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":90,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:29.368087342Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:38444"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:29.368232144Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"7dc00d64-09fa-4e67-b8cf-08d18ce05eac","remote_addr":"[fd00:1122:3344:101::c]:38444","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":60,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:29.376195153Z","hostname":"buskin","pid":1572,"sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:29.376214433Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"7618ab20-9498-402e-9d3a-a718b2bded8f","remote_addr":"[fd00:1122:3344:101::a]:64984","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":85,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:29.388847699Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"dd2c2175-c71f-4a02-b9bf-cde5eb1d13d2","remote_addr":"[fd00:1122:3344:101::a]:64984","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":218,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:29.404895649Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:49432"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:29.406650911Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"29d45244-6669-4047-9af2-a8b5039f291a","remote_addr":"[fd00:1122:3344:101::b]:49432","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1538,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:29.429468684Z","hostname":"buskin","pid":1572,"sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:29.429508714Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"112e62c0-fe22-4a6b-a10e-e5560e73ade2","remote_addr":"[fd00:1122:3344:101::c]:38444","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":2046,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:29.439977884Z","hostname":"buskin","pid":1572,"sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:29.440005685Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"c6298682-3e0f-487e-aa41-afd0eef0ded5","remote_addr":"[fd00:1122:3344:101::b]:49432","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":88,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:29.446216062Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:36626"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:29.446411224Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"0bbc683e-6030-4171-972d-2be913f08a11","remote_addr":"[fd00:1122:3344:101::c]:36626","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":151,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:29.449711995Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"6e50c580-f2cf-4584-a264-4cf25f9d2223","remote_addr":"[fd00:1122:3344:101::c]:38444","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":131,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:29.452799653Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"83df27fd-fa21-470b-8244-61a34b1bafb6","remote_addr":"[fd00:1122:3344:101::b]:49432","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":124,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:29.504779899Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:56556"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:29.504979391Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"5bf07006-7207-4e70-a9be-664494224ffc","remote_addr":"[fd00:1122:3344:101::a]:56556","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":143,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:29.526047933Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:36271"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:29.526141394Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:29.566495295Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:29.625675139Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"8e42937c-ac8d-4691-b64a-2b1842afb437","remote_addr":"[fd00:1122:3344:101::a]:36271","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":99564,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:29.625949123Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:62188"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:29.626022674Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:59390"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:29.626095275Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:57320"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:29.626347098Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"bb189e7f-5029-45ff-959e-da2a2c8f8161","remote_addr":"[fd00:1122:3344:101::a]:36271","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":331,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:29.626467739Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:29.626578431Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"91a1f119-ed82-4656-8ffa-c207fe95c7a0","remote_addr":"[fd00:1122:3344:101::b]:62188","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":303,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:29.667298206Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:29.723130129Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:29.72321856Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"c89b9bb6-7359-453d-8735-7b5c89042ed9","remote_addr":"[fd00:1122:3344:101::c]:57320","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":96881,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:29.726911796Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"79167855-8866-4741-8934-ae63c63d4282","remote_addr":"[fd00:1122:3344:101::c]:57320","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":693,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:29.75941562Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:29.818551594Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"b9b3cacc-c8ff-416b-a020-7e698d3ee838","remote_addr":"[fd00:1122:3344:101::b]:59390","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":192172,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:29.819567937Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"080b1c1c-0810-4188-af56-df1a9fcaa5fa","remote_addr":"[fd00:1122:3344:101::b]:59390","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":796,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:35.096740444Z","hostname":"buskin","pid":1572,"uri":"/b8a45863-ed9f-4b08-b97f-6fcba7a5a294","method":"GET","req_id":"82b7359a-e3ea-469d-bc38-e3bb565d2c05","remote_addr":"[fd00:1122:3344:101::d]:59149","local_addr":"[fd00:1122:3344:101::1]:34475","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","component":"producer-server","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":7879,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:36.094337447Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:40791"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:36.094438789Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"9dd8f981-91a5-4d4a-b048-2b03accf605d","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:40791","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":33,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:59.135569191Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:58638"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:59.135693483Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"3952dfed-4723-404b-ac00-b7f9b7b5b088","remote_addr":"[fd00:1122:3344:101::a]:58638","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":52,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:59.194033407Z","hostname":"buskin","pid":1572,"sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:59.194065598Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"70210388-aa68-458b-9142-8c4fa7b8a547","remote_addr":"[fd00:1122:3344:101::a]:58638","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":95,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:59.212442366Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"71219d7c-6235-4de4-8e46-5effa2040fbf","remote_addr":"[fd00:1122:3344:101::a]:58638","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1790,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:59.280270727Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:41446"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:59.280407458Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"77630930-a3a7-4a57-ae68-b96957cc8a0a","remote_addr":"[fd00:1122:3344:101::c]:41446","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":93,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:59.304792941Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:44661"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:59.305032824Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"e900dfa5-1927-4a62-9c6c-aa1c8143dd1f","remote_addr":"[fd00:1122:3344:101::c]:44661","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":166,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:59.334455189Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:62787"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:59.334659992Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"b4412ba6-e1ad-415b-8ebe-623f9832c574","remote_addr":"[fd00:1122:3344:101::b]:62787","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":63,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:59.356542904Z","hostname":"buskin","pid":1572,"sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:59.356562004Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"8271eb3b-5a90-4c34-905d-d2a02a45eda1","remote_addr":"[fd00:1122:3344:101::c]:41446","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":62,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:59.373292482Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"dd74f15e-e17d-476b-b895-f6c0fa2bab61","remote_addr":"[fd00:1122:3344:101::c]:41446","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":186,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:59.377611695Z","hostname":"buskin","pid":1572,"sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:59.377621565Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"e620148e-9f14-431e-96bf-d96af0087aff","remote_addr":"[fd00:1122:3344:101::b]:62787","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":41,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:59.389361291Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"c94b406f-5eb5-4fbf-a8a3-939ec714ff68","remote_addr":"[fd00:1122:3344:101::b]:62787","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":152,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:59.389948808Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:33374"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:59.390834399Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:59.437141964Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:59.503187774Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"f853105f-cf56-4499-a42d-cbae23b2e9d4","remote_addr":"[fd00:1122:3344:101::c]:33374","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":113168,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:59.503351016Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:54957"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:59.503416117Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:36854"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:59.503478998Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:63856"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:59.503762371Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:59.503866632Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"0a5f2936-5f2e-4a4d-bfec-895cabcf0d7a","remote_addr":"[fd00:1122:3344:101::c]:33374","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":346,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:59.507974243Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"4954fe46-1676-47cd-bab1-a71e0e810314","remote_addr":"[fd00:1122:3344:101::a]:54957","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":4184,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:59.517108727Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:58545"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:59.517303839Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"dac198bf-cfdd-4f5f-a09a-4b2966916c83","remote_addr":"[fd00:1122:3344:101::b]:58545","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":132,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:59.542196448Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:59.601530285Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:59.601593136Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"6cbadcbb-7b1d-49c0-92ba-c22d1ebb1255","remote_addr":"[fd00:1122:3344:101::a]:36854","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":97951,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:59.605311332Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"e14da7fd-c776-4ac1-a3b4-b778a7526f17","remote_addr":"[fd00:1122:3344:101::a]:36854","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1090,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:59.639090861Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:59.700398662Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"ab88c3da-82e3-45fd-b1fa-de04fdea949c","remote_addr":"[fd00:1122:3344:101::b]:63856","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":196674,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:57:59.700706466Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"eb9ee0dd-3fc7-47da-a372-0cf8f1708e59","remote_addr":"[fd00:1122:3344:101::b]:63856","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":132,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:04.835917861Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:49108"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:04.836125194Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"b00fc729-6363-4802-bab4-7f593649ced9","remote_addr":"[fd00:1122:3344:101::a]:49108","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":139,"response_code":"200"} {"msg":"Mapping virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:04.837669843Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:838","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(13442977) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:04.837701384Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"PUT","req_id":"17d388aa-762b-452a-9b3c-70a68b3a8fc1","remote_addr":"[fd00:1122:3344:101::a]:49108","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1412,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:05.003795645Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:54154"} {"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:05.012618385Z","hostname":"buskin","pid":1572,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:555","metadata":"InstanceMetadata { silo_id: 2050bd71-8ce2-4965-b903-b07fc427377b, project_id: 2e46faa0-ba1b-4b2e-9d42-9a1d1edf3ba4 }","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-10-12T03:58:05.000972341Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 641575e3-3293-4d69-8bf0-7455d660f873, kind: Instance { id: 9e1474b1-4580-4966-bf80-057e8706a109 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(13442977), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 641575e3-3293-4d69-8bf0-7455d660f873, kind: Instance { id: 9e1474b1-4580-4966-bf80-057e8706a109 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(13442977), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 641575e3-3293-4d69-8bf0-7455d660f873, kind: Instance { id: 9e1474b1-4580-4966-bf80-057e8706a109 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(13442977), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(13442977))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 641575e3-3293-4d69-8bf0-7455d660f873, kind: Instance { id: 9e1474b1-4580-4966-bf80-057e8706a109 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(13442977), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-c82af3a9245\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 7656ef46-ec12-4949-a766-1644f79e5aae, read_only_parent: Some(Volume { block_size: 512, id: 20022676-d771-4471-84f6-35fad14070ea, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: cf30f995-82c3-4bd1-87f7-def58b52e83b, key: Some(\"N1p3qreqOf1TWfoj7qTbtXTm5OvzuDDqPhOIoLtnDRs=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::1a]:19001\", \"[fd00:1122:3344:101::14]:19001\", \"[fd00:1122:3344:101::18]:19002\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 7656ef46-ec12-4949-a766-1644f79e5aae, key: Some(\"XDOKXQOH/E6Bx3Y0eCtP2W4mTw+eQbCiVwRAYOrvtyQ=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::18]:19003\", \"[fd00:1122:3344:101::12]:19000\", \"[fd00:1122:3344:101::16]:19000\"] } }] } }], boot_settings: Some(BootSettings { order: [BootOrderEntry { name: \"disk-c82af3a9245\" }] }), cloud_init_bytes: Some(..) }","migration_id":"None","propolis_id":"4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109"} {"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:05.012671456Z","hostname":"buskin","pid":1572,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:588","migration_id":"None","propolis_id":"4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109"} {"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:05.012679996Z","hostname":"buskin","pid":1572,"propolis_id":"4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","component":"InstanceManager","file":"sled-agent/src/instance.rs:1139","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 641575e3-3293-4d69-8bf0-7455d660f873, kind: Instance { id: 9e1474b1-4580-4966-bf80-057e8706a109 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(13442977), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 641575e3-3293-4d69-8bf0-7455d660f873, kind: Instance { id: 9e1474b1-4580-4966-bf80-057e8706a109 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(13442977), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 641575e3-3293-4d69-8bf0-7455d660f873, kind: Instance { id: 9e1474b1-4580-4966-bf80-057e8706a109 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(13442977), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(13442977))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 641575e3-3293-4d69-8bf0-7455d660f873, kind: Instance { id: 9e1474b1-4580-4966-bf80-057e8706a109 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(13442977), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-c82af3a9245\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 7656ef46-ec12-4949-a766-1644f79e5aae, read_only_parent: Some(Volume { block_size: 512, id: 20022676-d771-4471-84f6-35fad14070ea, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: cf30f995-82c3-4bd1-87f7-def58b52e83b, key: Some(\"N1p3qreqOf1TWfoj7qTbtXTm5OvzuDDqPhOIoLtnDRs=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::1a]:19001\", \"[fd00:1122:3344:101::14]:19001\", \"[fd00:1122:3344:101::18]:19002\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 7656ef46-ec12-4949-a766-1644f79e5aae, key: Some(\"XDOKXQOH/E6Bx3Y0eCtP2W4mTw+eQbCiVwRAYOrvtyQ=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::18]:19003\", \"[fd00:1122:3344:101::12]:19000\", \"[fd00:1122:3344:101::16]:19000\"] } }] } }], boot_settings: Some(BootSettings { order: [BootOrderEntry { name: \"disk-c82af3a9245\" }] }), cloud_init_bytes: Some(..) }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-10-12T03:58:05.000972341Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400, migration_id: None }","migration_id":"None","propolis_id":"4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:05.012798187Z","hostname":"buskin","pid":1572,"uri":"/vmms/4df0127b-3a24-46a3-990f-0c9ca848b366","method":"PUT","req_id":"e4d0628c-863f-4a35-a1ab-3e310f9493e6","remote_addr":"[fd00:1122:3344:101::a]:54154","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8901,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:05.097706991Z","hostname":"buskin","pid":1572,"uri":"/b8a45863-ed9f-4b08-b97f-6fcba7a5a294","method":"GET","req_id":"d9d0fd0e-5d32-4acd-8e7a-03a189905e3a","remote_addr":"[fd00:1122:3344:101::d]:59149","local_addr":"[fd00:1122:3344:101::1]:34475","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","component":"producer-server","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":8263,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:05.098562822Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:53921"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:05.100052811Z","hostname":"buskin","pid":1572,"uri":"/vmms/4df0127b-3a24-46a3-990f-0c9ca848b366/state","method":"PUT","req_id":"90d3c61e-c942-4240-bc81-98a9ec4f53e7","remote_addr":"[fd00:1122:3344:101::a]:53921","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1439,"response_code":"200"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:05.10324125Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 240, 0, 0]), slot: 0, vni: Vni { inner: 13442977 }, subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 } }) }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:05.104112831Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"Configuring new Omicron zone: oxz_propolis-server_4df0127b-3a24-46a3-990f-0c9ca848b366","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:05.144138068Z","hostname":"buskin","pid":1572,"instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","propolis_id":"4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","component":"InstanceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_propolis-server_4df0127b-3a24-46a3-990f-0c9ca848b366","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:05.167218574Z","hostname":"buskin","pid":1572,"instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","propolis_id":"4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","component":"InstanceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:05.20233251Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:33623"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:05.202514643Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"b74faede-1bf0-4fae-b524-6de4f3ee64ab","remote_addr":"[fd00:1122:3344:101::a]:33623","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":94,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:05.226143386Z","hostname":"buskin","pid":1572,"sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {641575e3-3293-4d69-8bf0-7455d660f873: {10.151.1.175: {cc91a9ac-1580-4452-829a-0c9deb3f5704}, 10.151.1.176: {cc91a9ac-1580-4452-829a-0c9deb3f5704}}} }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:06.130707125Z","hostname":"buskin","pid":1572,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:60636"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:06.130977438Z","hostname":"buskin","pid":1572,"uri":"/baseboard","method":"GET","req_id":"29003846-8000-442a-94db-15078f803547","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:60636","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":112,"response_code":"200"} {"msg":"Profile for oxz_propolis-server_4df0127b-3a24-46a3-990f-0c9ca848b366:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:06.465150085Z","hostname":"buskin","pid":1572,"instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","propolis_id":"4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_propolis-server_4df0127b-3a24-46a3-990f-0c9ca848b366 zone","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:06.465310338Z","hostname":"buskin","pid":1572,"zone":"oxz_propolis-server_4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","propolis_id":"4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_propolis-server_4df0127b-3a24-46a3-990f-0c9ca848b366\") failed: Property not found. retry in 26.154004ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:58:06.646565078Z","hostname":"buskin","pid":1572,"zone":"oxz_propolis-server_4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","propolis_id":"4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_4df0127b-3a24-46a3-990f-0c9ca848b366\") failed: Property not found. retry in 148.068319ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:58:06.682369112Z","hostname":"buskin","pid":1572,"zone":"oxz_propolis-server_4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","propolis_id":"4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_4df0127b-3a24-46a3-990f-0c9ca848b366\") failed: Property not found. retry in 138.14656ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:58:06.874489107Z","hostname":"buskin","pid":1572,"zone":"oxz_propolis-server_4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","propolis_id":"4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_4df0127b-3a24-46a3-990f-0c9ca848b366\") failed: Property not found. retry in 457.92256ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:58:07.031340234Z","hostname":"buskin","pid":1572,"zone":"oxz_propolis-server_4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","propolis_id":"4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_4df0127b-3a24-46a3-990f-0c9ca848b366\") failed: Property not found. retry in 629.063164ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:58:07.501715083Z","hostname":"buskin","pid":1572,"zone":"oxz_propolis-server_4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","propolis_id":"4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_4df0127b-3a24-46a3-990f-0c9ca848b366\") failed: Property not found. retry in 597.786262ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:58:08.142132103Z","hostname":"buskin","pid":1572,"zone":"oxz_propolis-server_4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","propolis_id":"4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_4df0127b-3a24-46a3-990f-0c9ca848b366\") failed: Property not found. retry in 645.632266ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:58:08.750898689Z","hostname":"buskin","pid":1572,"zone":"oxz_propolis-server_4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","propolis_id":"4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_4df0127b-3a24-46a3-990f-0c9ca848b366\") failed: Property not found. retry in 1.306316828s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:58:09.408324439Z","hostname":"buskin","pid":1572,"zone":"oxz_propolis-server_4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","propolis_id":"4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_4df0127b-3a24-46a3-990f-0c9ca848b366\") failed: Property not found. retry in 1.162330992s","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:58:10.726021396Z","hostname":"buskin","pid":1572,"zone":"oxz_propolis-server_4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","propolis_id":"4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Started propolis in zone: oxz_propolis-server_4df0127b-3a24-46a3-990f-0c9ca848b366","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:11.917872022Z","hostname":"buskin","pid":1572,"instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","propolis_id":"4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","component":"InstanceManager","file":"sled-agent/src/instance.rs:1633"} {"msg":"wait for service Some(\"oxz_propolis-server_4df0127b-3a24-46a3-990f-0c9ca848b366\") failed: Property not found. retry in 61.761719ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:58:11.926752382Z","hostname":"buskin","pid":1572,"instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","propolis_id":"4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_4df0127b-3a24-46a3-990f-0c9ca848b366\") failed: Property not found. retry in 80.431375ms","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:58:11.999196101Z","hostname":"buskin","pid":1572,"instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","propolis_id":"4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:12.090072859Z","hostname":"buskin","pid":1572,"instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","propolis_id":"4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","component":"InstanceManager","file":"sled-agent/src/instance.rs:1642"} {"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:12.129902934Z","hostname":"buskin","pid":1572,"instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","propolis_id":"4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","component":"InstanceManager","file":"sled-agent/src/instance.rs:1666"} {"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:12.269750479Z","hostname":"buskin","pid":1572,"instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","propolis_id":"4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","component":"InstanceManager","file":"sled-agent/src/instance.rs:835"} {"msg":"IGW mapping changed; external IPs refreshed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:12.270746702Z","hostname":"buskin","pid":1572,"sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:12.270818707Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"7ac07b51-b09b-4f91-9bda-9a65067760b8","remote_addr":"[fd00:1122:3344:101::a]:33623","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":7044873,"response_code":"204"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Creating), migration_in: None, migration_out: None, time: 2024-10-12T03:58:12.271245650Z }","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:12.271277197Z","hostname":"buskin","pid":1572,"instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","propolis_id":"4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","component":"InstanceManager","file":"sled-agent/src/instance.rs:685"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:12.271294541Z","hostname":"buskin","pid":1572,"instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","propolis_id":"4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","component":"InstanceManager","file":"sled-agent/src/instance.rs:707","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(3), time_updated: 2024-10-12T03:58:12.271245650Z }","propolis_id":"4df0127b-3a24-46a3-990f-0c9ca848b366"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:12.271302566Z","hostname":"buskin","pid":1572,"instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","propolis_id":"4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","component":"InstanceManager","file":"sled-agent/src/instance.rs:605","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Starting, gen: Generation(3), time_updated: 2024-10-12T03:58:12.271245650Z }, migration_in: None, migration_out: None }","propolis_id":"4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_in: None, migration_out: None, time: 2024-10-12T03:58:12.322366703Z }","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:12.322399763Z","hostname":"buskin","pid":1572,"instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","propolis_id":"4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","component":"InstanceManager","file":"sled-agent/src/instance.rs:685"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:12.322440344Z","hostname":"buskin","pid":1572,"instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","propolis_id":"4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","component":"InstanceManager","file":"sled-agent/src/instance.rs:707","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(4), time_updated: 2024-10-12T03:58:12.322366703Z }","propolis_id":"4df0127b-3a24-46a3-990f-0c9ca848b366"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:12.322457034Z","hostname":"buskin","pid":1572,"instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","propolis_id":"4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","component":"InstanceManager","file":"sled-agent/src/instance.rs:605","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Running, gen: Generation(4), time_updated: 2024-10-12T03:58:12.322366703Z }, migration_in: None, migration_out: None }","propolis_id":"4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:12.350596013Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"c21a8806-5a84-4303-99ca-b740525caa60","remote_addr":"[fd00:1122:3344:101::a]:33623","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":339,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:22.715967302Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:44721"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:22.71664318Z","hostname":"buskin","pid":1572,"uri":"/vmms/4df0127b-3a24-46a3-990f-0c9ca848b366/state","method":"PUT","req_id":"03e551f0-95de-4240-bb9e-fd7562d1454d","remote_addr":"[fd00:1122:3344:101::a]:44721","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":551,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_in: None, migration_out: None, time: 2024-10-12T03:58:22.716755212Z }","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:22.716779092Z","hostname":"buskin","pid":1572,"instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","propolis_id":"4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","component":"InstanceManager","file":"sled-agent/src/instance.rs:685"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:22.716789262Z","hostname":"buskin","pid":1572,"instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","propolis_id":"4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","component":"InstanceManager","file":"sled-agent/src/instance.rs:707","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(6), time_updated: 2024-10-12T03:58:22.716755212Z }","propolis_id":"4df0127b-3a24-46a3-990f-0c9ca848b366"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:22.716811742Z","hostname":"buskin","pid":1572,"instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","propolis_id":"4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","component":"InstanceManager","file":"sled-agent/src/instance.rs:605","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(6), time_updated: 2024-10-12T03:58:22.716755212Z }, migration_in: None, migration_out: None }","propolis_id":"4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_in: None, migration_out: None, time: 2024-10-12T03:58:22.726724115Z }","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:22.726739436Z","hostname":"buskin","pid":1572,"instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","propolis_id":"4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","component":"InstanceManager","file":"sled-agent/src/instance.rs:685"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:22.726790606Z","hostname":"buskin","pid":1572,"instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","propolis_id":"4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","component":"InstanceManager","file":"sled-agent/src/instance.rs:707","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-10-12T03:58:22.726724115Z }","propolis_id":"4df0127b-3a24-46a3-990f-0c9ca848b366"} {"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:22.726796936Z","hostname":"buskin","pid":1572,"instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","propolis_id":"4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","component":"InstanceManager","file":"sled-agent/src/instance.rs:721","propolis_id":"4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:22.726802616Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"target removed with queued samples","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:58:22.727411974Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:602","n_samples":6,"id":"10273551774547853836"} {"msg":"target removed with queued samples","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:58:22.727625467Z","hostname":"buskin","pid":1572,"component":"kstat-sampler-worker","sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:602","n_samples":6,"id":"7975133101321763214"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:22.72952415Z","hostname":"buskin","pid":1572,"component":"StorageManager","file":"sled-storage/src/manager.rs:451"} {"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:22.729564231Z","hostname":"buskin","pid":1572,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:339","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/464bcedf-19e0-4839-a12d-8d477420e762/debug/bundle/zone\", \"/pool/int/cd56c88b-70fb-4721-a4ca-a4de1b256c20/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/bed11bbb-e174-4475-bd7f-0211de5b5302/crypt/debug\", \"/pool/ext/2b84429f-2c85-4c1f-8f16-c6c336bc6cc2/crypt/debug\", \"/pool/ext/ddc852e6-47e6-4d51-a45f-adecc30d1fb3/crypt/debug\", \"/pool/ext/c0c860af-959d-47ac-8156-a6981a984705/crypt/debug\", \"/pool/ext/c590bfc3-f9d8-4665-9e2b-c8bbd2d3cd63/crypt/debug\", \"/pool/ext/a135191f-1186-46b0-a6c9-d425ec528829/crypt/debug\", \"/pool/ext/cd705dc8-a605-494a-9c72-bf21f8cc1d40/crypt/debug\", \"/pool/ext/42090923-c06f-4776-b57f-20fba0911cf3/crypt/debug\", \"/pool/ext/f95ba139-c738-48a5-85f9-774bb88e7e5a/crypt/debug\"] }","zone_name":"oxz_propolis-server_4df0127b-3a24-46a3-990f-0c9ca848b366"} {"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:23.361706038Z","hostname":"buskin","pid":1572,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1142","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_4df0127b-3a24-46a3-990f-0c9ca848b366\", bundle_id: 6849ed4e-5388-4fbf-8411-9fdcb94333ef }, time_created: 2024-10-12T03:58:22.729820334Z, version: 0, cause: TerminatedInstance }"} {"msg":"Halting and removing zone: oxz_propolis-server_4df0127b-3a24-46a3-990f-0c9ca848b366","v":0,"name":"SledAgent","level":40,"time":"2024-10-12T03:58:23.361798389Z","hostname":"buskin","pid":1572,"instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","propolis_id":"4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","component":"InstanceManager","file":"sled-agent/src/instance.rs:935"} {"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:25.109735378Z","hostname":"buskin","pid":1572,"instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","propolis_id":"4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","component":"InstanceManager","file":"illumos-utils/src/zone.rs:270"} {"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:25.127049481Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:964","id":"RouterId { vni: Vni(13442977), kind: System }"} {"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:25.127097154Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:964","id":"RouterId { vni: Vni(13442977), kind: Custom(V4(Ipv4Net { addr: 172.30.0.0, width: 22 })) }"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:25.127611014Z","hostname":"buskin","pid":1572,"instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","propolis_id":"4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","component":"InstanceManager","file":"sled-agent/src/instance.rs:605","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(8), time_updated: 2024-10-12T03:58:25.127581969Z }, migration_in: None, migration_out: None }","propolis_id":"4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:25.15669537Z","hostname":"buskin","pid":1572,"instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","propolis_id":"4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","component":"InstanceManager","file":"sled-agent/src/instance.rs:605","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(8), time_updated: 2024-10-12T03:58:25.127581969Z }, migration_in: None, migration_out: None }","propolis_id":"4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109"} {"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:25.160423437Z","hostname":"buskin","pid":1572,"instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","propolis_id":"4df0127b-3a24-46a3-990f-0c9ca848b366","instance_id":"9e1474b1-4580-4966-bf80-057e8706a109","component":"InstanceManager","file":"sled-agent/src/instance.rs:872"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:25.669577327Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:33981"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:25.669706568Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"d4cde817-fecc-40d8-aa96-5fcad1bce156","remote_addr":"[fd00:1122:3344:101::c]:33981","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":48,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:25.71095335Z","hostname":"buskin","pid":1572,"sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"IGW mapping changed; external IPs refreshed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:25.711038691Z","hostname":"buskin","pid":1572,"sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1205"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:25.711047431Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"94ad1896-df15-4d00-8de1-a2ed40501e19","remote_addr":"[fd00:1122:3344:101::c]:33981","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":146,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:25.721791245Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"77273dbb-83bb-447d-89c4-703d7ab23d3d","remote_addr":"[fd00:1122:3344:101::c]:33981","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":288,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:25.790978434Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:57027"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:25.791818354Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"73e0fde5-77fd-4854-9611-b2bbced5153f","remote_addr":"[fd00:1122:3344:101::c]:57027","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":775,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:27.004319596Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:45132"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:27.004534188Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"cdbc0932-af51-4c17-afd8-f716d9b2c913","remote_addr":"[fd00:1122:3344:101::a]:45132","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":141,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:27.058313336Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:59679"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:27.058393397Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"a8d55ebf-cbce-4f5a-a988-95a1dd2f455a","remote_addr":"[fd00:1122:3344:101::a]:59679","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":52,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:27.071383718Z","hostname":"buskin","pid":1572,"sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:27.071398989Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"ccb7db6c-ca5f-4f99-bd8f-2ad211d9382e","remote_addr":"[fd00:1122:3344:101::a]:59679","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":58,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:27.080166037Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"80e85ae9-7989-4a09-b39c-69f58f852bb0","remote_addr":"[fd00:1122:3344:101::a]:59679","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":189,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:27.622877174Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:54661"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:27.623510132Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"af2a69d9-529b-4e32-907b-c16ed6ce26ff","remote_addr":"[fd00:1122:3344:101::a]:54661","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":571,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:27.636862988Z","hostname":"buskin","pid":1572,"sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:27.636879438Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"76542e55-3c40-4f24-b16e-339a1c086391","remote_addr":"[fd00:1122:3344:101::a]:54661","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":63,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:27.645392134Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"2294b499-4dad-47c9-9a5b-4cf194ac8bfc","remote_addr":"[fd00:1122:3344:101::a]:54661","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":144,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.373227421Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:53864"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.373357503Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"fce965ee-8e66-4f6e-94c7-be925d4eb1a7","remote_addr":"[fd00:1122:3344:101::c]:53864","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":58,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.375900164Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:49116"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.376002355Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"2755ebfa-fbf3-4a6e-8a0e-417742b31d12","remote_addr":"[fd00:1122:3344:101::b]:49116","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":24,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.449433187Z","hostname":"buskin","pid":1572,"sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.449464117Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"2002ee09-f9cf-4ba6-8fd8-f96964d77a1c","remote_addr":"[fd00:1122:3344:101::c]:53864","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1698,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.453643669Z","hostname":"buskin","pid":1572,"sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.453663389Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"70c0b387-8859-47c9-8b98-1ccbabefd1fb","remote_addr":"[fd00:1122:3344:101::b]:49116","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":37,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.46662152Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"165c8c80-fa07-41cf-8e9f-fc7270674655","remote_addr":"[fd00:1122:3344:101::b]:49116","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":230,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.527228133Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:43023"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.527319544Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"GET","req_id":"5142410b-d838-4f0d-884e-3f48ba874d02","remote_addr":"[fd00:1122:3344:101::a]:43023","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":50,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.575962478Z","hostname":"buskin","pid":1572,"sled_id":"b8a45863-ed9f-4b08-b97f-6fcba7a5a294","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1179","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.575995018Z","hostname":"buskin","pid":1572,"uri":"/eip-gateways","method":"PUT","req_id":"8bc0dd57-59a2-4008-920b-4c71844a923c","remote_addr":"[fd00:1122:3344:101::a]:43023","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1283,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.587578302Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"93d5fbaf-88da-4f54-a840-c8bb467573d8","remote_addr":"[fd00:1122:3344:101::a]:43023","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":192,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.593186152Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:47551"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.593403624Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"3ebb0d62-f0b1-46df-9843-5d056daae687","remote_addr":"[fd00:1122:3344:101::c]:47551","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":102,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.60999721Z","hostname":"buskin","pid":1572,"uri":"/vpc-routes","method":"PUT","req_id":"d9304edf-6346-4fad-9b52-bba954fdbf67","remote_addr":"[fd00:1122:3344:101::c]:53864","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":321,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.659295256Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:48959"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.659378704Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.703100126Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.772327625Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"bb8c6faa-7cac-4da4-a016-a85b7e12501e","remote_addr":"[fd00:1122:3344:101::c]:48959","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":112954,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.772488947Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:51171"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.772553238Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:36564"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.772592879Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:56444"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.772931173Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"3daebeb5-811d-485f-b80d-bf7b25302839","remote_addr":"[fd00:1122:3344:101::c]:48959","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":305,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.772968893Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"43b7f692-44ce-4f75-a83d-4c56c7e4b0e5","remote_addr":"[fd00:1122:3344:101::b]:56444","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":277,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.773272217Z","hostname":"buskin","pid":1572,"uri":"/v2p","method":"GET","req_id":"46044ef6-07d7-45b8-bb46-7925438b65e1","remote_addr":"[fd00:1122:3344:101::a]:51171","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":449,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.774222869Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.816265331Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.823191797Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::c]:38841"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.824081208Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 5b00adda-efae-4057-bbdc-9e4e9e192aeb, kind: Service { id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b }, name: Name(\"external-dns-3ec2aea5-3e4e-4785-83a2-42c04a52438b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 106])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: bceffd56-91ce-4f4e-bc98-3823ea9f2972, kind: Service { id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2 }, name: Name(\"external-dns-0e7f7381-564c-4f95-8e8e-4ae9e12efae2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 231])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 704fe666-2d42-412a-9eca-2073bdc12aef, kind: Service { id: d24f88b5-85d4-41b0-9285-f812ad1f4772 }, name: Name(\"nexus-d24f88b5-85d4-41b0-9285-f812ad1f4772\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 98])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: bf4b340c-988d-42cb-b55b-9439e5c74f11, kind: Service { id: 055f90e5-f76f-48a0-93ae-4d52d8b374b4 }, name: Name(\"nexus-055f90e5-f76f-48a0-93ae-4d52d8b374b4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ce434cc7-d07f-4371-8836-7047706943dd, kind: Service { id: 4c95767b-a705-4932-acf6-7f257e60acef }, name: Name(\"nexus-4c95767b-a705-4932-acf6-7f257e60acef\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 44])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.824142599Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.824200559Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.824218129Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.8242487Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.82425856Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.82427917Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.824312261Z","hostname":"buskin","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"6bed7a55-1704-4267-aa4f-590617935638","remote_addr":"[fd00:1122:3344:101::c]:38841","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":985,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.827562001Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:36821"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.881928066Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4096"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.882007427Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"c0f0218d-e5e6-4c3f-a579-79046f29dc8e","remote_addr":"[fd00:1122:3344:101::a]:36564","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":108990,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.885886645Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"cf0ab72e-e265-44f5-aa3f-52597265de68","remote_addr":"[fd00:1122:3344:101::a]:36564","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":660,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.905876553Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::a]:47563"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.906153277Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 5b00adda-efae-4057-bbdc-9e4e9e192aeb, kind: Service { id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b }, name: Name(\"external-dns-3ec2aea5-3e4e-4785-83a2-42c04a52438b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 106])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: bceffd56-91ce-4f4e-bc98-3823ea9f2972, kind: Service { id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2 }, name: Name(\"external-dns-0e7f7381-564c-4f95-8e8e-4ae9e12efae2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 231])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 704fe666-2d42-412a-9eca-2073bdc12aef, kind: Service { id: d24f88b5-85d4-41b0-9285-f812ad1f4772 }, name: Name(\"nexus-d24f88b5-85d4-41b0-9285-f812ad1f4772\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 98])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: bf4b340c-988d-42cb-b55b-9439e5c74f11, kind: Service { id: 055f90e5-f76f-48a0-93ae-4d52d8b374b4 }, name: Name(\"nexus-055f90e5-f76f-48a0-93ae-4d52d8b374b4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ce434cc7-d07f-4371-8836-7047706943dd, kind: Service { id: 4c95767b-a705-4932-acf6-7f257e60acef }, name: Name(\"nexus-4c95767b-a705-4932-acf6-7f257e60acef\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 44])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.906210117Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.906250568Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.906260218Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.906281758Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.906293428Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.906309728Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.906332929Z","hostname":"buskin","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"786b5d7a-7179-4e64-80d8-a8e5bed3d2db","remote_addr":"[fd00:1122:3344:101::a]:47563","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":373,"response_code":"204"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.919248399Z","hostname":"buskin","pid":1572,"component":"ServiceManager","file":"sled-agent/src/services.rs:4120"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.934734181Z","hostname":"buskin","pid":1572,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:1105","remote_addr":"[fd00:1122:3344:101::b]:42353"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.935814695Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 5b00adda-efae-4057-bbdc-9e4e9e192aeb, kind: Service { id: 3ec2aea5-3e4e-4785-83a2-42c04a52438b }, name: Name(\"external-dns-3ec2aea5-3e4e-4785-83a2-42c04a52438b\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 106])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: bceffd56-91ce-4f4e-bc98-3823ea9f2972, kind: Service { id: 0e7f7381-564c-4f95-8e8e-4ae9e12efae2 }, name: Name(\"external-dns-0e7f7381-564c-4f95-8e8e-4ae9e12efae2\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 250, 231])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 704fe666-2d42-412a-9eca-2073bdc12aef, kind: Service { id: d24f88b5-85d4-41b0-9285-f812ad1f4772 }, name: Name(\"nexus-d24f88b5-85d4-41b0-9285-f812ad1f4772\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 98])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: bf4b340c-988d-42cb-b55b-9439e5c74f11, kind: Service { id: 055f90e5-f76f-48a0-93ae-4d52d8b374b4 }, name: Name(\"nexus-055f90e5-f76f-48a0-93ae-4d52d8b374b4\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 173])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: ce434cc7-d07f-4371-8836-7047706943dd, kind: Service { id: 4c95767b-a705-4932-acf6-7f257e60acef }, name: Name(\"nexus-4c95767b-a705-4932-acf6-7f257e60acef\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 44])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.935872415Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.935925516Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.935942016Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.935977177Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.935990937Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.936011817Z","hostname":"buskin","pid":1572,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.936024407Z","hostname":"buskin","pid":1572,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"f41df0eb-d892-4a43-b8dd-334ba97b2277","remote_addr":"[fd00:1122:3344:101::b]:42353","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":1107,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.977141718Z","hostname":"buskin","pid":1572,"uri":"/switch-ports","method":"POST","req_id":"8add89db-fff2-4bd0-a609-e83a6deba590","remote_addr":"[fd00:1122:3344:101::b]:36821","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":149519,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-10-12T03:58:29.977448762Z","hostname":"buskin","pid":1572,"uri":"/network-bootstore-config","method":"GET","req_id":"f2e30fee-9244-4877-8a1c-514bdef1e56e","remote_addr":"[fd00:1122:3344:101::b]:36821","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.12.0/src/server.rs:950","latency_us":139,"response_code":"200"}