[ Jan 15 02:23:10 Enabled. ] [ Jan 15 02:23:10 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ Jan 15 02:23:10 Method "start" exited with status 0. ] [ Jan 15 02:23:10 Rereading configuration. ] [ Jan 15 02:23:10 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:10.759051707Z","hostname":"sock","pid":1566,"file":"illumos-utils/src/running_zone.rs:113"} {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:10.759205251Z","hostname":"sock","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:10.814220786Z","hostname":"sock","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:10.814264387Z","hostname":"sock","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:10.929539487Z","hostname":"sock","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:10.93853921Z","hostname":"sock","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:10.946527786Z","hostname":"sock","pid":1566,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:11.555495018Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fdb0:18c0:4d0d:a02a::, width: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:11.692372211Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:126"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:11.692404912Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:138"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:11.692512055Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:151"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:11.692608938Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:173","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:11.692619238Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:520"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:11.692631348Z","hostname":"sock","pid":1566,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:717"} {"msg":"Generated i86pc baseboard Pc { identifier: \"sock\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:11.69563507Z","hostname":"sock","pid":1566,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:624"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:11.69564269Z","hostname":"sock","pid":1566,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:766"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:11.69565243Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:187"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:11.69566102Z","hostname":"sock","pid":1566,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:825"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:11.695684351Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_0.vdev"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:11.695744952Z","hostname":"sock","pid":1566,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"None"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:11.695842655Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:11.695866796Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:11.695881966Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:11.708158517Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:11.708192608Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:378","id":"04e15bc9-6d7d-41db-92b0-ff86f85636e0 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:11.808850625Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 04e15bc9-6d7d-41db-92b0-ff86f85636e0 (zpool), kind: Internal }"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.157340308Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.157374399Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_1.vdev"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.157380299Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.157392099Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.157400119Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.15741578Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.1574199Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.166062913Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.166081674Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:378","id":"f20ff590-e55a-4339-aef1-379a1d2d3fc3 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.255287971Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: f20ff590-e55a-4339-aef1-379a1d2d3fc3 (zpool), kind: Internal }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.612760287Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_0.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.612802658Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 04e15bc9-6d7d-41db-92b0-ff86f85636e0 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: f20ff590-e55a-4339-aef1-379a1d2d3fc3 (zpool), kind: Internal } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.612825879Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.612831429Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.61284736Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.61285841Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.612899461Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_1.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.612926991Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.612932172Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 04e15bc9-6d7d-41db-92b0-ff86f85636e0 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: f20ff590-e55a-4339-aef1-379a1d2d3fc3 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.612947022Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.612951182Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.612967373Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.613038145Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_2.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.613042725Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 04e15bc9-6d7d-41db-92b0-ff86f85636e0 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: f20ff590-e55a-4339-aef1-379a1d2d3fc3 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.613057055Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.613060535Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.613063955Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.613068425Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.613071866Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_3.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.613074926Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 04e15bc9-6d7d-41db-92b0-ff86f85636e0 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: f20ff590-e55a-4339-aef1-379a1d2d3fc3 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.613087996Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.613099456Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.613102866Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.613105956Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.613110027Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_4.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.613113137Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 04e15bc9-6d7d-41db-92b0-ff86f85636e0 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: f20ff590-e55a-4339-aef1-379a1d2d3fc3 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.613128387Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.613132447Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.613135877Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.613139087Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.613142977Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_5.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.613145928Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 04e15bc9-6d7d-41db-92b0-ff86f85636e0 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: f20ff590-e55a-4339-aef1-379a1d2d3fc3 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.613174578Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.613178048Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.613181558Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.613187309Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.613190869Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_6.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.613194329Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 04e15bc9-6d7d-41db-92b0-ff86f85636e0 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: f20ff590-e55a-4339-aef1-379a1d2d3fc3 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.61322049Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.61322448Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.61322815Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.61323267Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.61323648Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_7.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.61324053Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 04e15bc9-6d7d-41db-92b0-ff86f85636e0 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: f20ff590-e55a-4339-aef1-379a1d2d3fc3 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.613267511Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.613280401Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.613284221Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.613287971Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.613291461Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_8.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.613294532Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 04e15bc9-6d7d-41db-92b0-ff86f85636e0 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: f20ff590-e55a-4339-aef1-379a1d2d3fc3 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.613314672Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.613318182Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.613321842Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.613324982Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.613357333Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:97"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.613362163Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 04e15bc9-6d7d-41db-92b0-ff86f85636e0 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: f20ff590-e55a-4339-aef1-379a1d2d3fc3 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.613393284Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:99"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.613396934Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.613400434Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.613403635Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.613452376Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:211"} {"msg":"No ledger in /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.613529488Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.613548128Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.613561059Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.613577149Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.613587009Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.61359128Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.613637681Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:216"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.613644791Z","hostname":"sock","pid":1566,"file":"sled-agent/src/long_running_tasks.rs:231"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.625421199Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:737"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.6254471Z","hostname":"sock","pid":1566,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:186","time_to_next_cleanup":"599.999942919s","next_cleanup":"Instant { tv_sec: 3679, tv_nsec: 662321125 }"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.625513371Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3962"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.625518401Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"No ledger in /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.625567053Z","hostname":"sock","pid":1566,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.625576583Z","hostname":"sock","pid":1566,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.628488262Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.630142096Z","hostname":"sock","pid":1566,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:51","local_addr":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.2.164,\n last: 10.151.2.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.2.164,\n 10.151.2.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.2.174,\n infra_ip_last: 10.151.2.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.2.100,\n vlan_id: None,\n rib_priority: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.2.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: Some(\n None,\n ),\n bgp_peers: [],\n autoneg: false,\n lldp: None,\n tx_eq: None,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n}","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.630617289Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1204"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.630625219Z","hostname":"sock","pid":1566,"component":"DnsResolver","component":"RSS","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.630683401Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.630718402Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"No ledger in /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/rss-started.marker","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.630753463Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/rss-started.marker","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.630776893Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.630785573Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.630800064Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"RSS not previously run. Creating plans.","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.630803914Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1253"} {"msg":"Writing ledger to /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/.rss-started.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.630813084Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/.rss-started.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.63102257Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.631176714Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1285"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.631181114Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:66"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:a02a::1","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.631184514Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:39"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.631217065Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1316"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.631280317Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.631291057Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.631463572Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.631609306Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.631685288Z","hostname":"sock","pid":1566,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: ce66c06a-7d54-47a6-860d-59da68080fee (sled), rack_id: 0c4f466f-f877-43e3-8a10-8b20ac7ed608, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.63253024Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4259"} {"msg":"Starting switch zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.632595592Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4659"} {"msg":"Loaded keys and certs","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.637099674Z","hostname":"sock","pid":1566,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:18c0:4d0d:a02a::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/client.rs:50"} {"msg":"Certificate chain verified successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.643118126Z","hostname":"sock","pid":1566,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:18c0:4d0d:a02a::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:307"} {"msg":"Signature verified successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.643223299Z","hostname":"sock","pid":1566,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:18c0:4d0d:a02a::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:376"} {"msg":"Certificate chain verified successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.647362311Z","hostname":"sock","pid":1566,"component":"SledAgentSprocketsServer","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:307"} {"msg":"Signature verified successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.647422282Z","hostname":"sock","pid":1566,"component":"SledAgentSprocketsServer","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:376"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.647505955Z","hostname":"sock","pid":1566,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:62701","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:74"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: ce66c06a-7d54-47a6-860d-59da68080fee (sled), rack_id: 0c4f466f-f877-43e3-8a10-8b20ac7ed608, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.647631368Z","hostname":"sock","pid":1566,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:362"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.647646599Z","hostname":"sock","pid":1566,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:377"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.647685329Z","hostname":"sock","pid":1566,"file":"sled-agent/src/server.rs:44"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.64769239Z","hostname":"sock","pid":1566,"component":"DnsResolver","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.647760102Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:101::, width: 64 }"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.647769892Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.647773952Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:813"} {"msg":"No ledger in /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.647810833Z","hostname":"sock","pid":1566,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.647817793Z","hostname":"sock","pid":1566,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.647828193Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:802"} {"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.647833084Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:830"} {"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.647837224Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:596"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.647882425Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"} {"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.647890555Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:701","result":"DisksManagementResult { status: [] }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.664542455Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.685944932Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.703025913Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.709000794Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:412"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.709038085Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.709042935Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:424"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.709087686Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 111.515985ms)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.712423707Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"53567dfa-4cde-4068-a89b-153fe9a48e95\", \"content-length\": \"124\", \"date\": \"Wed, 15 Jan 2025 02:23:12 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"53567dfa-4cde-4068-a89b-153fe9a48e95\" }"} {"msg":"attempting to create encrypted zvol: name=\"oxi_04e15bc9-6d7d-41db-92b0-ff86f85636e0/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.722409286Z","hostname":"sock","pid":1566,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"successfully created encrypted zvol: name=\"oxi_04e15bc9-6d7d-41db-92b0-ff86f85636e0/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.746040374Z","hostname":"sock","pid":1566,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_04e15bc9-6d7d-41db-92b0-ff86f85636e0/swap\"","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.746093825Z","hostname":"sock","pid":1566,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.751387038Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:439"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.751401558Z","hostname":"sock","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:129"} {"msg":"Ensuring dataset oxi_04e15bc9-6d7d-41db-92b0-ff86f85636e0/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.751406618Z","hostname":"sock","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:139"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.820432211Z","hostname":"sock","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:174"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.82448557Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 81.625346ms)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.825108157Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8f7af6cc-8445-4d04-917c-81bc1192e772\", \"content-length\": \"124\", \"date\": \"Wed, 15 Jan 2025 02:23:12 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8f7af6cc-8445-4d04-917c-81bc1192e772\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.907192972Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 120.346639ms)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:12.907607573Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"dd8b900b-38a1-41e9-8674-0d58fe0b82b6\", \"content-length\": \"124\", \"date\": \"Wed, 15 Jan 2025 02:23:12 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"dd8b900b-38a1-41e9-8674-0d58fe0b82b6\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:13.028517165Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 119.7698ms)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:13.029091951Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"05841650-26b2-48c6-bcae-cfe27b3dfb2d\", \"content-length\": \"124\", \"date\": \"Wed, 15 Jan 2025 02:23:12 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"05841650-26b2-48c6-bcae-cfe27b3dfb2d\" }"} {"msg":"Mounting oxi_04e15bc9-6d7d-41db-92b0-ff86f85636e0/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:13.04017485Z","hostname":"sock","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:182"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:13.052557514Z","hostname":"sock","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:190"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:13.052645067Z","hostname":"sock","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:190"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:13.052684068Z","hostname":"sock","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:190"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:13.052716858Z","hostname":"sock","pid":1566,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:196"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:13.149935872Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 221.032278ms)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:13.15058275Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"98d8ad84-172b-4896-9c96-d521903dabdb\", \"content-length\": \"124\", \"date\": \"Wed, 15 Jan 2025 02:23:12 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"98d8ad84-172b-4896-9c96-d521903dabdb\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:13.372482807Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 145.311963ms)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:13.373281259Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"23c53c9c-b0af-4be7-83fe-bf0c8fe5b132\", \"content-length\": \"124\", \"date\": \"Wed, 15 Jan 2025 02:23:12 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"23c53c9c-b0af-4be7-83fe-bf0c8fe5b132\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:13.519733971Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 329.481286ms)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:13.520357688Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"735737bc-5244-4cb1-b35c-a09b55983e03\", \"content-length\": \"124\", \"date\": \"Wed, 15 Jan 2025 02:23:12 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"735737bc-5244-4cb1-b35c-a09b55983e03\" }"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:13.783107438Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:93"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:13.789254524Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:63949","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327","component":"producer-server","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:13.789295525Z","hostname":"sock","pid":1566,"component":"internal-dns-resolver","file":"internal-dns/resolver/src/resolver.rs:111","component":"producer-server","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:13.789354616Z","hostname":"sock","pid":1566,"file":"oximeter/producer/src/lib.rs:279","component":"producer-server","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"oximeter/producer/src/lib.rs:279","interval":"30s","address":"[fd00:1122:3344:101::1]:63949","producer_id":"ce66c06a-7d54-47a6-860d-59da68080fee"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:13.79021641Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"238.928126ms"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:13.791492114Z","hostname":"sock","pid":1566,"component":"VmmReservoirManager","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:13.791500284Z","hostname":"sock","pid":1566,"component":"VmmReservoirManager","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:13.791541926Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:921","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:13.791547426Z","hostname":"sock","pid":1566,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:13.850534417Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:14.039783273Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"315.39351ms"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:14.357195209Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"1.177836533s"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:15.536032399Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"2.586917902s"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:16.420901506Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2683"} {"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:16.420923676Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2928"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:16.420928826Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2553"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:16.420933126Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2600"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:16.420968287Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3009"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:16.420976328Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3054"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:16.423236819Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2853"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:16.424453872Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:2594"} {"msg":"Profile for oxz_switch:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:16.424586235Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_switch zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:16.42475811Z","hostname":"sock","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 26.709792ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:16.816268414Z","hostname":"sock","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 56.872567ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:17.05071628Z","hostname":"sock","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 139.793359ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:17.238441256Z","hostname":"sock","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 402.479543ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:17.440684283Z","hostname":"sock","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 873.900382ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:18.040810047Z","hostname":"sock","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:18.124626019Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"2.645513601s"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 782.390784ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:19.057990485Z","hostname":"sock","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.162402252s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:19.900819998Z","hostname":"sock","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:20.771977415Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"11.125125766s"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.059641314s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:21.12274894Z","hostname":"sock","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ce66c06a-7d54-47a6-860d-59da68080fee)","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:22.542629114Z","hostname":"sock","pid":1566,"component":"NexusNotifierTask","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/nexus.rs:527"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:22.572709226Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:22.573141078Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12348","component":"dropshot (Repo Depot)","component":"ArtifactStore","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:22.574400402Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:22.574417592Z","hostname":"sock","pid":1566,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:432"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:22.574428832Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Writing ledger to /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:22.574442033Z","hostname":"sock","pid":1566,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:22.574619858Z","hostname":"sock","pid":1566,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:22.574777562Z","hostname":"sock","pid":1566,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:22.574842454Z","hostname":"sock","pid":1566,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV2 { rack_subnet: Ipv6Net { addr: fd00:1122:3344:100::, width: 56 }, infra_ip_first: 10.151.2.174, infra_ip_last: 10.151.2.174, ports: [PortConfigV2 { routes: [RouteConfig { destination: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), nexthop: 10.151.2.100, vlan_id: None, rib_priority: None }], addresses: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: Some(None), bgp_peers: [], autoneg: false, lldp: None, tx_eq: None }], bgp: [], bfd: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:22.574858214Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3962"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:22.574861964Z","hostname":"sock","pid":1566,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:62701","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:81"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:22.746353062Z","hostname":"sock","pid":1566,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:117","peer":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:22.746430224Z","hostname":"sock","pid":1566,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:252","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:22.752002244Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4283","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:22.752010535Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4299"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:22.752014985Z","hostname":"sock","pid":1566,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:534"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:22.992165565Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:42821"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:22.992513504Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:22.992520344Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"No ledger in /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:22.992560125Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:22.992588376Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:22.992711959Z","hostname":"sock","pid":1566,"uri":"/inventory","method":"GET","req_id":"9611c3ad-d2ad-4f9a-8d51-03823f19db22","remote_addr":"[fd00:1122:3344:101::1]:42821","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":300,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:23.023832869Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:40117"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:23.024023234Z","hostname":"sock","pid":1566,"uri":"/sled-role","method":"GET","req_id":"37a1211c-8c84-4eae-be0d-a2f5a05d0dd7","remote_addr":"[fd00:1122:3344:101::1]:40117","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":71,"response_code":"200"} {"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { generation: Generation(2), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: 5159126c-9e12-452b-bbff-cb9e350cd14d (physical_disk), pool_id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: 622dbf5d-3394-49f5-97fe-7669c6beb731 (physical_disk), pool_id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: 2be4fc44-4fd1-4313-a2d9-06a022e6cac0 (physical_disk), pool_id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 6825fc9a-8ecc-436c-9914-6a84f4e83a37 (physical_disk), pool_id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: e4126dd3-13f7-4311-be1a-c9bb9500e47e (physical_disk), pool_id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: 73efa030-117a-404c-b3f7-485378a920dc (physical_disk), pool_id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: b15cb332-fda4-43ec-b0a0-01f79c43351a (physical_disk), pool_id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 98f08ec8-ef4f-48ea-9849-84bf762ef4f4 (physical_disk), pool_id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: 7345914d-ec5e-4941-af7b-f7a172c14bcc (physical_disk), pool_id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool) }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:23.0616539Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:385"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:23.061809454Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:44201"} {"msg":"physical disks ensure","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:23.062084671Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:893"} {"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(2), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: 5159126c-9e12-452b-bbff-cb9e350cd14d (physical_disk), pool_id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: 622dbf5d-3394-49f5-97fe-7669c6beb731 (physical_disk), pool_id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: 2be4fc44-4fd1-4313-a2d9-06a022e6cac0 (physical_disk), pool_id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 6825fc9a-8ecc-436c-9914-6a84f4e83a37 (physical_disk), pool_id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: e4126dd3-13f7-4311-be1a-c9bb9500e47e (physical_disk), pool_id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: 73efa030-117a-404c-b3f7-485378a920dc (physical_disk), pool_id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: b15cb332-fda4-43ec-b0a0-01f79c43351a (physical_disk), pool_id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 98f08ec8-ef4f-48ea-9849-84bf762ef4f4 (physical_disk), pool_id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: 7345914d-ec5e-4941-af7b-f7a172c14bcc (physical_disk), pool_id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:23.062130353Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"No ledger in /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:23.062143403Z","hostname":"sock","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:23.062152363Z","hostname":"sock","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:23.062157403Z","hostname":"sock","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1298"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:23.062182534Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:23.062188394Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:23.062203725Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:23.062207525Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:23.223920189Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:23.223949329Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool)"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ce66c06a-7d54-47a6-860d-59da68080fee)","v":0,"name":"SledAgent","level":50,"time":"2025-01-15T02:23:23.650033317Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-15T02:23:23.898470951Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"413521614313442029"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-15T02:23:23.898586654Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"17478185358689782290"} {"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.090354878Z","hostname":"sock","pid":1566,"component":"VmmReservoirManager","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.118765675Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.127256664Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.127333626Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.127344636Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.127362477Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.127369997Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.12745819Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.12746474Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.163774739Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_57cb94f0-4114-4bd3-970a-a0245b825f9d/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.172223578Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_57cb94f0-4114-4bd3-970a-a0245b825f9d","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.314567079Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.322902784Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.322945445Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.322954405Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.322962455Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.322969545Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.332317558Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.332360289Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.425539053Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.433743344Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.433797946Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.433804616Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.433823917Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.433832057Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.43393584Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.43394234Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.470502576Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_27bb8d18-eb2e-4bb2-b35c-c031637c236f/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.478490282Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_27bb8d18-eb2e-4bb2-b35c-c031637c236f","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.616856736Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.624716828Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.624768869Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.624778539Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.62478399Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.62478975Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.633444153Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.633469234Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"87e49e9a-4ab3-4574-b920-463506a8978e (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.727654066Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","zpool":"ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.736235067Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.736268428Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.736276348Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.736303879Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.736313369Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.736437672Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.736466703Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ce66c06a-7d54-47a6-860d-59da68080fee)","v":0,"name":"SledAgent","level":50,"time":"2025-01-15T02:23:24.739952197Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.772374402Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_87e49e9a-4ab3-4574-b920-463506a8978e/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.780359368Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_87e49e9a-4ab3-4574-b920-463506a8978e","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.916706967Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.924803526Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.924833586Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.924841356Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.924845847Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.924850687Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.934889828Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:24.934911928Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.030669482Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","zpool":"ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.039317455Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.039342585Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.039348006Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.039379237Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.039399857Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.03952101Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.039528581Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.073748934Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_b471fd39-9806-47fa-8ddb-0ca2f42aaf5d/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.081569305Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_b471fd39-9806-47fa-8ddb-0ca2f42aaf5d","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.223817914Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.231786738Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.231817669Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.23182764Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.23185588Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.23186216Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.241914222Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.241932212Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.337630505Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","zpool":"ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.345908048Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.345929369Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.345934159Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.34596723Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.345991Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.346086503Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.346093773Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.381385505Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_03ea8739-b702-4a4b-b400-0745bd6b4f96/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.38934674Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_03ea8739-b702-4a4b-b400-0745bd6b4f96","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.523420828Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.531067954Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.531110696Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.531118066Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.531126356Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.531133576Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.540456848Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.540483468Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"d8acd544-80aa-4e82-b375-eb038fe27547 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.637423195Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","zpool":"ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.644978858Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.645008299Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.645014299Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.64503328Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.64504119Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.645183504Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.645190374Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.679792698Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_d8acd544-80aa-4e82-b375-eb038fe27547/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.687578028Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ce66c06a-7d54-47a6-860d-59da68080fee)","v":0,"name":"SledAgent","level":50,"time":"2025-01-15T02:23:25.74139067Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Looking for unencrypted datasets in oxp_d8acd544-80aa-4e82-b375-eb038fe27547","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.821330587Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.829457456Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.829500727Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.829508248Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.829515468Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.829521538Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.839917209Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.83995626Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.935176449Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","zpool":"ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.943235557Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.943282418Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.943290608Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.943332029Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.94334245Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.943471263Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.943479603Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.978639562Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_0aaa1466-164a-4540-ae41-ab17e9fc6bc8/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:25.986714809Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_0aaa1466-164a-4540-ae41-ab17e9fc6bc8","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.125904985Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.133653715Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.133677775Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.133697606Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.133702326Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.133707356Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.143407058Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.143433038Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.236098799Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","zpool":"ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.244165747Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.244193128Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.244199428Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.244217118Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.244224548Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.244353372Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.244359112Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.280719623Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_56c8079e-8d4e-4506-b6ff-2f0a5efab320/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.289660474Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"Looking for unencrypted datasets in oxp_56c8079e-8d4e-4506-b6ff-2f0a5efab320","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.428007908Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.435955872Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.435989993Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.436000983Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.436008833Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.436037894Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.446372143Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.446407754Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"276e9391-34d6-4a99-bd91-776ecada771e (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.543829523Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","zpool":"ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.551672695Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:239","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.551767017Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:241","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.551778708Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:245","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.551825009Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:247","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.551838519Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.552024004Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.552043035Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:259"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.58597929Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_276e9391-34d6-4a99-bd91-776ecada771e/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.594949742Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:302"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ce66c06a-7d54-47a6-860d-59da68080fee)","v":0,"name":"SledAgent","level":50,"time":"2025-01-15T02:23:26.743922593Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Looking for unencrypted datasets in oxp_276e9391-34d6-4a99-bd91-776ecada771e","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.747964671Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/dataset.rs:388"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.756554553Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.756586564Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Writing ledger to /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.756665596Z","hostname":"sock","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.756780269Z","hostname":"sock","pid":1566,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(2), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 04e15bc9-6d7d-41db-92b0-ff86f85636e0 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: f20ff590-e55a-4339-aef1-379a1d2d3fc3 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.756825061Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.756830411Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Writing ledger to /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.756931724Z","hostname":"sock","pid":1566,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"physical disks ensure: Updated storage","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.75715471Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:897"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.75716726Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"physical disks ensure: Propagating new generation of disks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.75717627Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:923","generation":"Generation(2)"} {"msg":"physical disks ensure: Updated storage monitor","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.75718062Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:928"} {"msg":"physical disks ensure: Updated zone bundler","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.75718562Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:933"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.757215731Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:122","gen":"Generation(2)"} {"msg":"physical disks ensure: Updated probes","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.757222921Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:939"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.757263133Z","hostname":"sock","pid":1566,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:799","gen":"Generation(2)"} {"msg":"physical disks ensure: Updated instances","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.757268273Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:944"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.757332034Z","hostname":"sock","pid":1566,"uri":"/omicron-physical-disks","method":"PUT","req_id":"8d0d4f79-63a7-49e8-8f79-b2edcf47f4cb","remote_addr":"[fd00:1122:3344:101::1]:44201","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":3695388,"response_code":"200"} {"msg":"attempting to set up sled's datasets: DatasetsConfig { generation: Generation(1), datasets: {0289b49f-f46b-48bd-8877-22ab22e20ade (dataset): DatasetConfig { id: 0289b49f-f46b-48bd-8877-22ab22e20ade (dataset), name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 06e3a5aa-51e3-40cd-8681-80ad502a87f0 (dataset): DatasetConfig { id: 06e3a5aa-51e3-40cd-8681-80ad502a87f0 (dataset), name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 0cf2d24e-167f-4c85-b8b3-f81c61fb0a2e (dataset): DatasetConfig { id: 0cf2d24e-167f-4c85-b8b3-f81c61fb0a2e (dataset), name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 0ed23841-25ee-4ce1-9a3f-a0eaa44b2968 (dataset): DatasetConfig { id: 0ed23841-25ee-4ce1-9a3f-a0eaa44b2968 (dataset), name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 10d093dd-a650-4250-9a52-625c59d8429d (dataset): DatasetConfig { id: 10d093dd-a650-4250-9a52-625c59d8429d (dataset), name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: TransientZone { name: \"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 12d38b84-38e4-497b-a99f-cc965216a4f9 (dataset): DatasetConfig { id: 12d38b84-38e4-497b-a99f-cc965216a4f9 (dataset), name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 144bc7a7-ac1d-4608-8dbd-5ac10e4a3401 (dataset): DatasetConfig { id: 144bc7a7-ac1d-4608-8dbd-5ac10e4a3401 (dataset), name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_788bf241-055b-45bf-8d67-e865c6324e46\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 15d06938-5ed1-495c-8d9e-f1215d6707f0 (dataset): DatasetConfig { id: 15d06938-5ed1-495c-8d9e-f1215d6707f0 (dataset), name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 192d5e96-8014-4e3c-af48-71d4c3fb693f (dataset): DatasetConfig { id: 192d5e96-8014-4e3c-af48-71d4c3fb693f (dataset), name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 1c6376c2-3da7-460c-9e3b-69c6b433d692 (dataset): DatasetConfig { id: 1c6376c2-3da7-460c-9e3b-69c6b433d692 (dataset), name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2301231b-0d94-4a71-be13-10f124de2f45 (dataset): DatasetConfig { id: 2301231b-0d94-4a71-be13-10f124de2f45 (dataset), name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 24691edf-c48b-437a-bc02-60c286d7b8cf (dataset): DatasetConfig { id: 24691edf-c48b-437a-bc02-60c286d7b8cf (dataset), name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2b41717f-a5eb-4dbd-807e-f3e09cc06f74 (dataset): DatasetConfig { id: 2b41717f-a5eb-4dbd-807e-f3e09cc06f74 (dataset), name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 2f63adaa-76db-41cc-b893-1e1fa8ec4023 (dataset): DatasetConfig { id: 2f63adaa-76db-41cc-b893-1e1fa8ec4023 (dataset), name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: TransientZone { name: \"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 33a8e0a2-3f66-4ad6-b8bb-7051cfd0cbaa (dataset): DatasetConfig { id: 33a8e0a2-3f66-4ad6-b8bb-7051cfd0cbaa (dataset), name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 38b42a5e-7027-4de1-8ac2-3ea59c97440e (dataset): DatasetConfig { id: 38b42a5e-7027-4de1-8ac2-3ea59c97440e (dataset), name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 3a8b3fc6-ba51-4788-a2d6-52b005519793 (dataset): DatasetConfig { id: 3a8b3fc6-ba51-4788-a2d6-52b005519793 (dataset), name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 404c68d7-7db9-4b17-95df-80f4686b2c54 (dataset): DatasetConfig { id: 404c68d7-7db9-4b17-95df-80f4686b2c54 (dataset), name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 4086dcee-b755-4ef2-bdb3-b67201420856 (dataset): DatasetConfig { id: 4086dcee-b755-4ef2-bdb3-b67201420856 (dataset), name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 42184284-ec0c-4eab-ac7a-eab1166ce5e0 (dataset): DatasetConfig { id: 42184284-ec0c-4eab-ac7a-eab1166ce5e0 (dataset), name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 46e1fc42-a569-4dfb-abe7-316a667df8cf (dataset): DatasetConfig { id: 46e1fc42-a569-4dfb-abe7-316a667df8cf (dataset), name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 48370fb2-c834-4a33-8784-b4aeaa582c74 (dataset): DatasetConfig { id: 48370fb2-c834-4a33-8784-b4aeaa582c74 (dataset), name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4b0ee27b-8da6-4c04-8d51-bb5197e541f3 (dataset): DatasetConfig { id: 4b0ee27b-8da6-4c04-8d51-bb5197e541f3 (dataset), name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4c507982-7a99-43bc-bea8-37d473cad711 (dataset): DatasetConfig { id: 4c507982-7a99-43bc-bea8-37d473cad711 (dataset), name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 5111907f-1a66-483b-8265-031220c3551a (dataset): DatasetConfig { id: 5111907f-1a66-483b-8265-031220c3551a (dataset), name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: TransientZone { name: \"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 52d75b9d-b91e-42ce-9720-11020ff487a7 (dataset): DatasetConfig { id: 52d75b9d-b91e-42ce-9720-11020ff487a7 (dataset), name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: TransientZone { name: \"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 54e1beab-aba6-4fdc-8db5-a7a24fedf96a (dataset): DatasetConfig { id: 54e1beab-aba6-4fdc-8db5-a7a24fedf96a (dataset), name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 57ff1210-2d75-43c2-9f20-cb3a7d5d54ca (dataset): DatasetConfig { id: 57ff1210-2d75-43c2-9f20-cb3a7d5d54ca (dataset), name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 5aa196c1-9ae5-462e-b25f-60a6d097b874 (dataset): DatasetConfig { id: 5aa196c1-9ae5-462e-b25f-60a6d097b874 (dataset), name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 63ca0d5d-bbd4-4fbb-8fb7-6665d51c71d8 (dataset): DatasetConfig { id: 63ca0d5d-bbd4-4fbb-8fb7-6665d51c71d8 (dataset), name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 64011c35-1c58-4535-9959-04a87d4dc601 (dataset): DatasetConfig { id: 64011c35-1c58-4535-9959-04a87d4dc601 (dataset), name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_a4d0b75d-0c67-4224-af19-0510665c8775\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 648b6836-9354-41f1-b7f0-6163e19a5888 (dataset): DatasetConfig { id: 648b6836-9354-41f1-b7f0-6163e19a5888 (dataset), name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 692ad8cc-32a5-494a-9b91-dfb00956045b (dataset): DatasetConfig { id: 692ad8cc-32a5-494a-9b91-dfb00956045b (dataset), name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 69f3dbe4-c82b-4e19-8171-254683ee8c41 (dataset): DatasetConfig { id: 69f3dbe4-c82b-4e19-8171-254683ee8c41 (dataset), name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_a6c98ed9-62e0-43ae-9066-c1d4ee8e0596\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 6b6000c5-54ae-425e-81d0-945d32df1d9b (dataset): DatasetConfig { id: 6b6000c5-54ae-425e-81d0-945d32df1d9b (dataset), name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 6ca20e7b-3d78-4f22-b166-09085a169b88 (dataset): DatasetConfig { id: 6ca20e7b-3d78-4f22-b166-09085a169b88 (dataset), name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 714a3d51-7354-4136-ab84-4ab3553f9a14 (dataset): DatasetConfig { id: 714a3d51-7354-4136-ab84-4ab3553f9a14 (dataset), name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 72acf285-2bf1-463d-9388-034fc3e2d535 (dataset): DatasetConfig { id: 72acf285-2bf1-463d-9388-034fc3e2d535 (dataset), name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 7bfaac18-ed9d-4f63-9286-a22adc5ea76b (dataset): DatasetConfig { id: 7bfaac18-ed9d-4f63-9286-a22adc5ea76b (dataset), name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 807b7e75-1ba3-40fd-a9d6-9fd5b21a8c44 (dataset): DatasetConfig { id: 807b7e75-1ba3-40fd-a9d6-9fd5b21a8c44 (dataset), name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 838ed889-1e6a-4e9a-ab3b-8cccb456c92a (dataset): DatasetConfig { id: 838ed889-1e6a-4e9a-ab3b-8cccb456c92a (dataset), name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 8de08a8a-c957-4bba-ad69-05a8bb6cc49a (dataset): DatasetConfig { id: 8de08a8a-c957-4bba-ad69-05a8bb6cc49a (dataset), name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 93c7f407-f119-4375-a48b-5cd3c5de6365 (dataset): DatasetConfig { id: 93c7f407-f119-4375-a48b-5cd3c5de6365 (dataset), name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 947c5cb9-d823-4a46-92e3-258c98b3fbda (dataset): DatasetConfig { id: 947c5cb9-d823-4a46-92e3-258c98b3fbda (dataset), name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 94f97465-3c59-442d-b314-9251e12768a4 (dataset): DatasetConfig { id: 94f97465-3c59-442d-b314-9251e12768a4 (dataset), name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 99303814-f72e-4aa4-9696-b7384618360e (dataset): DatasetConfig { id: 99303814-f72e-4aa4-9696-b7384618360e (dataset), name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 9aee71c2-1bd5-4322-8835-62409d2c83da (dataset): DatasetConfig { id: 9aee71c2-1bd5-4322-8835-62409d2c83da (dataset), name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 9d5a4900-0fa4-48fb-8edd-929d848ce656 (dataset): DatasetConfig { id: 9d5a4900-0fa4-48fb-8edd-929d848ce656 (dataset), name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_44ac331a-da63-47e3-a636-214c3ca86b22\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 9ee24c3a-ffe5-4fa3-adcb-cb33f95b7e5e (dataset): DatasetConfig { id: 9ee24c3a-ffe5-4fa3-adcb-cb33f95b7e5e (dataset), name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, acb545cd-f9ac-4def-9b0a-c4360da7d2e5 (dataset): DatasetConfig { id: acb545cd-f9ac-4def-9b0a-c4360da7d2e5 (dataset), name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b78ba258-0fb2-4de9-a2ab-575bb62d4d34 (dataset): DatasetConfig { id: b78ba258-0fb2-4de9-a2ab-575bb62d4d34 (dataset), name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, bd10e7f6-07a9-4f3d-bacf-0ca102692080 (dataset): DatasetConfig { id: bd10e7f6-07a9-4f3d-bacf-0ca102692080 (dataset), name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c05328a6-b132-4291-b7ad-f43c76bcc2e9 (dataset): DatasetConfig { id: c05328a6-b132-4291-b7ad-f43c76bcc2e9 (dataset), name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, c600fb25-6a61-4125-adb2-bdd91fa47b63 (dataset): DatasetConfig { id: c600fb25-6a61-4125-adb2-bdd91fa47b63 (dataset), name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, cacea80f-5c49-4409-a465-5fa9b8fafb8e (dataset): DatasetConfig { id: cacea80f-5c49-4409-a465-5fa9b8fafb8e (dataset), name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ce2c4352-d2fa-4418-ad3d-e549862eeda1 (dataset): DatasetConfig { id: ce2c4352-d2fa-4418-ad3d-e549862eeda1 (dataset), name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d085d928-418e-4c85-b94d-9f8328f05a85 (dataset): DatasetConfig { id: d085d928-418e-4c85-b94d-9f8328f05a85 (dataset), name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, dcbe6f41-d5ce-4a2e-af3f-5216a427ff46 (dataset): DatasetConfig { id: dcbe6f41-d5ce-4a2e-af3f-5216a427ff46 (dataset), name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_77bd825f-cbb2-4b31-ba9f-d69e9e4cff59\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, dfdad261-72fe-491f-9719-63b4b577069d (dataset): DatasetConfig { id: dfdad261-72fe-491f-9719-63b4b577069d (dataset), name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e17342f9-9bf6-4cf6-a640-e521037d96b7 (dataset): DatasetConfig { id: e17342f9-9bf6-4cf6-a640-e521037d96b7 (dataset), name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: TransientZone { name: \"oxz_ntp_fd5ec85c-1d4a-43a3-bd64-3bf287ecded3\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e2ae6ff8-e428-490b-9473-8a87ce5b820e (dataset): DatasetConfig { id: e2ae6ff8-e428-490b-9473-8a87ce5b820e (dataset), name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_82a65d0c-f391-4b1f-9c11-e3a397904f10\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e81e6c51-0411-4b29-9c35-755633d8c74e (dataset): DatasetConfig { id: e81e6c51-0411-4b29-9c35-755633d8c74e (dataset), name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f059e034-b8f8-4950-84f5-75e333282237 (dataset): DatasetConfig { id: f059e034-b8f8-4950-84f5-75e333282237 (dataset), name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f28efc5c-c2d2-4edc-9ca6-887b5393e9c5 (dataset): DatasetConfig { id: f28efc5c-c2d2-4edc-9ca6-887b5393e9c5 (dataset), name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_0e94e881-2bab-4d68-8315-741d3cd1814a\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f42b4c2a-381d-404f-932f-4065926dc088 (dataset): DatasetConfig { id: f42b4c2a-381d-404f-932f-4065926dc088 (dataset), name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, fa13626c-d453-4db3-aab8-037ff9ed27cc (dataset): DatasetConfig { id: fa13626c-d453-4db3-aab8-037ff9ed27cc (dataset), name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }} }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.789374599Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:460"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.789478622Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:63500"} {"msg":"datasets ensure","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.790088328Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:859"} {"msg":"Received DatasetsEnsure { config: DatasetsConfig { generation: Generation(1), datasets: {0289b49f-f46b-48bd-8877-22ab22e20ade (dataset): DatasetConfig { id: 0289b49f-f46b-48bd-8877-22ab22e20ade (dataset), name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 06e3a5aa-51e3-40cd-8681-80ad502a87f0 (dataset): DatasetConfig { id: 06e3a5aa-51e3-40cd-8681-80ad502a87f0 (dataset), name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 0cf2d24e-167f-4c85-b8b3-f81c61fb0a2e (dataset): DatasetConfig { id: 0cf2d24e-167f-4c85-b8b3-f81c61fb0a2e (dataset), name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 0ed23841-25ee-4ce1-9a3f-a0eaa44b2968 (dataset): DatasetConfig { id: 0ed23841-25ee-4ce1-9a3f-a0eaa44b2968 (dataset), name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 10d093dd-a650-4250-9a52-625c59d8429d (dataset): DatasetConfig { id: 10d093dd-a650-4250-9a52-625c59d8429d (dataset), name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: TransientZone { name: \"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 12d38b84-38e4-497b-a99f-cc965216a4f9 (dataset): DatasetConfig { id: 12d38b84-38e4-497b-a99f-cc965216a4f9 (dataset), name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 144bc7a7-ac1d-4608-8dbd-5ac10e4a3401 (dataset): DatasetConfig { id: 144bc7a7-ac1d-4608-8dbd-5ac10e4a3401 (dataset), name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_788bf241-055b-45bf-8d67-e865c6324e46\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 15d06938-5ed1-495c-8d9e-f1215d6707f0 (dataset): DatasetConfig { id: 15d06938-5ed1-495c-8d9e-f1215d6707f0 (dataset), name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 192d5e96-8014-4e3c-af48-71d4c3fb693f (dataset): DatasetConfig { id: 192d5e96-8014-4e3c-af48-71d4c3fb693f (dataset), name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 1c6376c2-3da7-460c-9e3b-69c6b433d692 (dataset): DatasetConfig { id: 1c6376c2-3da7-460c-9e3b-69c6b433d692 (dataset), name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2301231b-0d94-4a71-be13-10f124de2f45 (dataset): DatasetConfig { id: 2301231b-0d94-4a71-be13-10f124de2f45 (dataset), name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 24691edf-c48b-437a-bc02-60c286d7b8cf (dataset): DatasetConfig { id: 24691edf-c48b-437a-bc02-60c286d7b8cf (dataset), name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2b41717f-a5eb-4dbd-807e-f3e09cc06f74 (dataset): DatasetConfig { id: 2b41717f-a5eb-4dbd-807e-f3e09cc06f74 (dataset), name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 2f63adaa-76db-41cc-b893-1e1fa8ec4023 (dataset): DatasetConfig { id: 2f63adaa-76db-41cc-b893-1e1fa8ec4023 (dataset), name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: TransientZone { name: \"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 33a8e0a2-3f66-4ad6-b8bb-7051cfd0cbaa (dataset): DatasetConfig { id: 33a8e0a2-3f66-4ad6-b8bb-7051cfd0cbaa (dataset), name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 38b42a5e-7027-4de1-8ac2-3ea59c97440e (dataset): DatasetConfig { id: 38b42a5e-7027-4de1-8ac2-3ea59c97440e (dataset), name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 3a8b3fc6-ba51-4788-a2d6-52b005519793 (dataset): DatasetConfig { id: 3a8b3fc6-ba51-4788-a2d6-52b005519793 (dataset), name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 404c68d7-7db9-4b17-95df-80f4686b2c54 (dataset): DatasetConfig { id: 404c68d7-7db9-4b17-95df-80f4686b2c54 (dataset), name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 4086dcee-b755-4ef2-bdb3-b67201420856 (dataset): DatasetConfig { id: 4086dcee-b755-4ef2-bdb3-b67201420856 (dataset), name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 42184284-ec0c-4eab-ac7a-eab1166ce5e0 (dataset): DatasetConfig { id: 42184284-ec0c-4eab-ac7a-eab1166ce5e0 (dataset), name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 46e1fc42-a569-4dfb-abe7-316a667df8cf (dataset): DatasetConfig { id: 46e1fc42-a569-4dfb-abe7-316a667df8cf (dataset), name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 48370fb2-c834-4a33-8784-b4aeaa582c74 (dataset): DatasetConfig { id: 48370fb2-c834-4a33-8784-b4aeaa582c74 (dataset), name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4b0ee27b-8da6-4c04-8d51-bb5197e541f3 (dataset): DatasetConfig { id: 4b0ee27b-8da6-4c04-8d51-bb5197e541f3 (dataset), name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4c507982-7a99-43bc-bea8-37d473cad711 (dataset): DatasetConfig { id: 4c507982-7a99-43bc-bea8-37d473cad711 (dataset), name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 5111907f-1a66-483b-8265-031220c3551a (dataset): DatasetConfig { id: 5111907f-1a66-483b-8265-031220c3551a (dataset), name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: TransientZone { name: \"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 52d75b9d-b91e-42ce-9720-11020ff487a7 (dataset): DatasetConfig { id: 52d75b9d-b91e-42ce-9720-11020ff487a7 (dataset), name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: TransientZone { name: \"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 54e1beab-aba6-4fdc-8db5-a7a24fedf96a (dataset): DatasetConfig { id: 54e1beab-aba6-4fdc-8db5-a7a24fedf96a (dataset), name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 57ff1210-2d75-43c2-9f20-cb3a7d5d54ca (dataset): DatasetConfig { id: 57ff1210-2d75-43c2-9f20-cb3a7d5d54ca (dataset), name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 5aa196c1-9ae5-462e-b25f-60a6d097b874 (dataset): DatasetConfig { id: 5aa196c1-9ae5-462e-b25f-60a6d097b874 (dataset), name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 63ca0d5d-bbd4-4fbb-8fb7-6665d51c71d8 (dataset): DatasetConfig { id: 63ca0d5d-bbd4-4fbb-8fb7-6665d51c71d8 (dataset), name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 64011c35-1c58-4535-9959-04a87d4dc601 (dataset): DatasetConfig { id: 64011c35-1c58-4535-9959-04a87d4dc601 (dataset), name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_a4d0b75d-0c67-4224-af19-0510665c8775\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 648b6836-9354-41f1-b7f0-6163e19a5888 (dataset): DatasetConfig { id: 648b6836-9354-41f1-b7f0-6163e19a5888 (dataset), name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 692ad8cc-32a5-494a-9b91-dfb00956045b (dataset): DatasetConfig { id: 692ad8cc-32a5-494a-9b91-dfb00956045b (dataset), name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 69f3dbe4-c82b-4e19-8171-254683ee8c41 (dataset): DatasetConfig { id: 69f3dbe4-c82b-4e19-8171-254683ee8c41 (dataset), name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_a6c98ed9-62e0-43ae-9066-c1d4ee8e0596\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 6b6000c5-54ae-425e-81d0-945d32df1d9b (dataset): DatasetConfig { id: 6b6000c5-54ae-425e-81d0-945d32df1d9b (dataset), name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 6ca20e7b-3d78-4f22-b166-09085a169b88 (dataset): DatasetConfig { id: 6ca20e7b-3d78-4f22-b166-09085a169b88 (dataset), name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 714a3d51-7354-4136-ab84-4ab3553f9a14 (dataset): DatasetConfig { id: 714a3d51-7354-4136-ab84-4ab3553f9a14 (dataset), name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 72acf285-2bf1-463d-9388-034fc3e2d535 (dataset): DatasetConfig { id: 72acf285-2bf1-463d-9388-034fc3e2d535 (dataset), name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 7bfaac18-ed9d-4f63-9286-a22adc5ea76b (dataset): DatasetConfig { id: 7bfaac18-ed9d-4f63-9286-a22adc5ea76b (dataset), name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 807b7e75-1ba3-40fd-a9d6-9fd5b21a8c44 (dataset): DatasetConfig { id: 807b7e75-1ba3-40fd-a9d6-9fd5b21a8c44 (dataset), name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 838ed889-1e6a-4e9a-ab3b-8cccb456c92a (dataset): DatasetConfig { id: 838ed889-1e6a-4e9a-ab3b-8cccb456c92a (dataset), name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 8de08a8a-c957-4bba-ad69-05a8bb6cc49a (dataset): DatasetConfig { id: 8de08a8a-c957-4bba-ad69-05a8bb6cc49a (dataset), name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 93c7f407-f119-4375-a48b-5cd3c5de6365 (dataset): DatasetConfig { id: 93c7f407-f119-4375-a48b-5cd3c5de6365 (dataset), name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 947c5cb9-d823-4a46-92e3-258c98b3fbda (dataset): DatasetConfig { id: 947c5cb9-d823-4a46-92e3-258c98b3fbda (dataset), name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 94f97465-3c59-442d-b314-9251e12768a4 (dataset): DatasetConfig { id: 94f97465-3c59-442d-b314-9251e12768a4 (dataset), name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 99303814-f72e-4aa4-9696-b7384618360e (dataset): DatasetConfig { id: 99303814-f72e-4aa4-9696-b7384618360e (dataset), name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 9aee71c2-1bd5-4322-8835-62409d2c83da (dataset): DatasetConfig { id: 9aee71c2-1bd5-4322-8835-62409d2c83da (dataset), name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 9d5a4900-0fa4-48fb-8edd-929d848ce656 (dataset): DatasetConfig { id: 9d5a4900-0fa4-48fb-8edd-929d848ce656 (dataset), name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_44ac331a-da63-47e3-a636-214c3ca86b22\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 9ee24c3a-ffe5-4fa3-adcb-cb33f95b7e5e (dataset): DatasetConfig { id: 9ee24c3a-ffe5-4fa3-adcb-cb33f95b7e5e (dataset), name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, acb545cd-f9ac-4def-9b0a-c4360da7d2e5 (dataset): DatasetConfig { id: acb545cd-f9ac-4def-9b0a-c4360da7d2e5 (dataset), name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, b78ba258-0fb2-4de9-a2ab-575bb62d4d34 (dataset): DatasetConfig { id: b78ba258-0fb2-4de9-a2ab-575bb62d4d34 (dataset), name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, bd10e7f6-07a9-4f3d-bacf-0ca102692080 (dataset): DatasetConfig { id: bd10e7f6-07a9-4f3d-bacf-0ca102692080 (dataset), name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c05328a6-b132-4291-b7ad-f43c76bcc2e9 (dataset): DatasetConfig { id: c05328a6-b132-4291-b7ad-f43c76bcc2e9 (dataset), name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, c600fb25-6a61-4125-adb2-bdd91fa47b63 (dataset): DatasetConfig { id: c600fb25-6a61-4125-adb2-bdd91fa47b63 (dataset), name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, cacea80f-5c49-4409-a465-5fa9b8fafb8e (dataset): DatasetConfig { id: cacea80f-5c49-4409-a465-5fa9b8fafb8e (dataset), name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ce2c4352-d2fa-4418-ad3d-e549862eeda1 (dataset): DatasetConfig { id: ce2c4352-d2fa-4418-ad3d-e549862eeda1 (dataset), name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d085d928-418e-4c85-b94d-9f8328f05a85 (dataset): DatasetConfig { id: d085d928-418e-4c85-b94d-9f8328f05a85 (dataset), name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, dcbe6f41-d5ce-4a2e-af3f-5216a427ff46 (dataset): DatasetConfig { id: dcbe6f41-d5ce-4a2e-af3f-5216a427ff46 (dataset), name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_77bd825f-cbb2-4b31-ba9f-d69e9e4cff59\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, dfdad261-72fe-491f-9719-63b4b577069d (dataset): DatasetConfig { id: dfdad261-72fe-491f-9719-63b4b577069d (dataset), name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e17342f9-9bf6-4cf6-a640-e521037d96b7 (dataset): DatasetConfig { id: e17342f9-9bf6-4cf6-a640-e521037d96b7 (dataset), name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: TransientZone { name: \"oxz_ntp_fd5ec85c-1d4a-43a3-bd64-3bf287ecded3\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e2ae6ff8-e428-490b-9473-8a87ce5b820e (dataset): DatasetConfig { id: e2ae6ff8-e428-490b-9473-8a87ce5b820e (dataset), name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_82a65d0c-f391-4b1f-9c11-e3a397904f10\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e81e6c51-0411-4b29-9c35-755633d8c74e (dataset): DatasetConfig { id: e81e6c51-0411-4b29-9c35-755633d8c74e (dataset), name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f059e034-b8f8-4950-84f5-75e333282237 (dataset): DatasetConfig { id: f059e034-b8f8-4950-84f5-75e333282237 (dataset), name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f28efc5c-c2d2-4edc-9ca6-887b5393e9c5 (dataset): DatasetConfig { id: f28efc5c-c2d2-4edc-9ca6-887b5393e9c5 (dataset), name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_0e94e881-2bab-4d68-8315-741d3cd1814a\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f42b4c2a-381d-404f-932f-4065926dc088 (dataset): DatasetConfig { id: f42b4c2a-381d-404f-932f-4065926dc088 (dataset), name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, fa13626c-d453-4db3-aab8-037ff9ed27cc (dataset): DatasetConfig { id: fa13626c-d453-4db3-aab8-037ff9ed27cc (dataset), name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }} }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.790292044Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"No ledger in /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.790297764Z","hostname":"sock","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.790304254Z","hostname":"sock","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested datasets', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.790307784Z","hostname":"sock","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:909"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.80757067Z","hostname":"sock","pid":1566,"name":"oxp_87e49e9a-4ab3-4574-b920-463506a8978e/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.807600851Z","hostname":"sock","pid":1566,"name":"oxp_87e49e9a-4ab3-4574-b920-463506a8978e/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.807605851Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_87e49e9a-4ab3-4574-b920-463506a8978e/crypt/internal_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.85687042Z","hostname":"sock","pid":1566,"name":"oxp_0aaa1466-164a-4540-ae41-ab17e9fc6bc8/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.856900341Z","hostname":"sock","pid":1566,"name":"oxp_0aaa1466-164a-4540-ae41-ab17e9fc6bc8/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.856905091Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/0aaa1466-164a-4540-ae41-ab17e9fc6bc8/crypt/debug\"), full_name: \"oxp_0aaa1466-164a-4540-ae41-ab17e9fc6bc8/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.890135518Z","hostname":"sock","pid":1566,"name":"oxp_b471fd39-9806-47fa-8ddb-0ca2f42aaf5d/crypt/zone/oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.890169089Z","hostname":"sock","pid":1566,"name":"oxp_b471fd39-9806-47fa-8ddb-0ca2f42aaf5d/crypt/zone/oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.890173229Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/b471fd39-9806-47fa-8ddb-0ca2f42aaf5d/crypt/zone/oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\"), full_name: \"oxp_b471fd39-9806-47fa-8ddb-0ca2f42aaf5d/crypt/zone/oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.951431812Z","hostname":"sock","pid":1566,"name":"oxp_27bb8d18-eb2e-4bb2-b35c-c031637c236f/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.951468353Z","hostname":"sock","pid":1566,"name":"oxp_27bb8d18-eb2e-4bb2-b35c-c031637c236f/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.951477303Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/27bb8d18-eb2e-4bb2-b35c-c031637c236f/crypt/zone\"), full_name: \"oxp_27bb8d18-eb2e-4bb2-b35c-c031637c236f/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.978840261Z","hostname":"sock","pid":1566,"name":"oxp_27bb8d18-eb2e-4bb2-b35c-c031637c236f/crypt/zone/oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.978872022Z","hostname":"sock","pid":1566,"name":"oxp_27bb8d18-eb2e-4bb2-b35c-c031637c236f/crypt/zone/oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:26.978875962Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/27bb8d18-eb2e-4bb2-b35c-c031637c236f/crypt/zone/oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a\"), full_name: \"oxp_27bb8d18-eb2e-4bb2-b35c-c031637c236f/crypt/zone/oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.039156968Z","hostname":"sock","pid":1566,"name":"oxp_d8acd544-80aa-4e82-b375-eb038fe27547/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.039195389Z","hostname":"sock","pid":1566,"name":"oxp_d8acd544-80aa-4e82-b375-eb038fe27547/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.03920021Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/d8acd544-80aa-4e82-b375-eb038fe27547/crypt/debug\"), full_name: \"oxp_d8acd544-80aa-4e82-b375-eb038fe27547/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.066903897Z","hostname":"sock","pid":1566,"name":"oxp_87e49e9a-4ab3-4574-b920-463506a8978e/crypt/zone/oxz_cockroachdb_788bf241-055b-45bf-8d67-e865c6324e46","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.066942398Z","hostname":"sock","pid":1566,"name":"oxp_87e49e9a-4ab3-4574-b920-463506a8978e/crypt/zone/oxz_cockroachdb_788bf241-055b-45bf-8d67-e865c6324e46","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.066946698Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/87e49e9a-4ab3-4574-b920-463506a8978e/crypt/zone/oxz_cockroachdb_788bf241-055b-45bf-8d67-e865c6324e46\"), full_name: \"oxp_87e49e9a-4ab3-4574-b920-463506a8978e/crypt/zone/oxz_cockroachdb_788bf241-055b-45bf-8d67-e865c6324e46\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.126514396Z","hostname":"sock","pid":1566,"name":"oxp_03ea8739-b702-4a4b-b400-0745bd6b4f96/crypt/zone/oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.126554627Z","hostname":"sock","pid":1566,"name":"oxp_03ea8739-b702-4a4b-b400-0745bd6b4f96/crypt/zone/oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.126558997Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/03ea8739-b702-4a4b-b400-0745bd6b4f96/crypt/zone/oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f\"), full_name: \"oxp_03ea8739-b702-4a4b-b400-0745bd6b4f96/crypt/zone/oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.186425482Z","hostname":"sock","pid":1566,"name":"oxp_56c8079e-8d4e-4506-b6ff-2f0a5efab320/crypt/zone/oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.186460003Z","hostname":"sock","pid":1566,"name":"oxp_56c8079e-8d4e-4506-b6ff-2f0a5efab320/crypt/zone/oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.186464183Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/56c8079e-8d4e-4506-b6ff-2f0a5efab320/crypt/zone/oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b\"), full_name: \"oxp_56c8079e-8d4e-4506-b6ff-2f0a5efab320/crypt/zone/oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.246989987Z","hostname":"sock","pid":1566,"name":"oxp_0aaa1466-164a-4540-ae41-ab17e9fc6bc8/crypt/zone/oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.247024668Z","hostname":"sock","pid":1566,"name":"oxp_0aaa1466-164a-4540-ae41-ab17e9fc6bc8/crypt/zone/oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.247028778Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/0aaa1466-164a-4540-ae41-ab17e9fc6bc8/crypt/zone/oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45\"), full_name: \"oxp_0aaa1466-164a-4540-ae41-ab17e9fc6bc8/crypt/zone/oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.307068528Z","hostname":"sock","pid":1566,"name":"oxp_0aaa1466-164a-4540-ae41-ab17e9fc6bc8/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.307103859Z","hostname":"sock","pid":1566,"name":"oxp_0aaa1466-164a-4540-ae41-ab17e9fc6bc8/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.307108029Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_0aaa1466-164a-4540-ae41-ab17e9fc6bc8/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.355352151Z","hostname":"sock","pid":1566,"name":"oxp_57cb94f0-4114-4bd3-970a-a0245b825f9d/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.355384202Z","hostname":"sock","pid":1566,"name":"oxp_57cb94f0-4114-4bd3-970a-a0245b825f9d/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.355388752Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_57cb94f0-4114-4bd3-970a-a0245b825f9d/crypt/internal_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.405607167Z","hostname":"sock","pid":1566,"name":"oxp_56c8079e-8d4e-4506-b6ff-2f0a5efab320/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.405639378Z","hostname":"sock","pid":1566,"name":"oxp_56c8079e-8d4e-4506-b6ff-2f0a5efab320/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.405643548Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/56c8079e-8d4e-4506-b6ff-2f0a5efab320/crypt/debug\"), full_name: \"oxp_56c8079e-8d4e-4506-b6ff-2f0a5efab320/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.435798222Z","hostname":"sock","pid":1566,"name":"oxp_57cb94f0-4114-4bd3-970a-a0245b825f9d/crypt/zone/oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.435830632Z","hostname":"sock","pid":1566,"name":"oxp_57cb94f0-4114-4bd3-970a-a0245b825f9d/crypt/zone/oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.435834693Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/57cb94f0-4114-4bd3-970a-a0245b825f9d/crypt/zone/oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301\"), full_name: \"oxp_57cb94f0-4114-4bd3-970a-a0245b825f9d/crypt/zone/oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.496938812Z","hostname":"sock","pid":1566,"name":"oxp_27bb8d18-eb2e-4bb2-b35c-c031637c236f/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.496973613Z","hostname":"sock","pid":1566,"name":"oxp_27bb8d18-eb2e-4bb2-b35c-c031637c236f/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.496977662Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_27bb8d18-eb2e-4bb2-b35c-c031637c236f/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.547890427Z","hostname":"sock","pid":1566,"name":"oxp_27bb8d18-eb2e-4bb2-b35c-c031637c236f/crypt/external_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.547916577Z","hostname":"sock","pid":1566,"name":"oxp_27bb8d18-eb2e-4bb2-b35c-c031637c236f/crypt/external_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.547920647Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_27bb8d18-eb2e-4bb2-b35c-c031637c236f/crypt/external_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.597916026Z","hostname":"sock","pid":1566,"name":"oxp_0aaa1466-164a-4540-ae41-ab17e9fc6bc8/crypt/zone/oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.597945947Z","hostname":"sock","pid":1566,"name":"oxp_0aaa1466-164a-4540-ae41-ab17e9fc6bc8/crypt/zone/oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.597950237Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/0aaa1466-164a-4540-ae41-ab17e9fc6bc8/crypt/zone/oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca\"), full_name: \"oxp_0aaa1466-164a-4540-ae41-ab17e9fc6bc8/crypt/zone/oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.658590454Z","hostname":"sock","pid":1566,"name":"oxp_276e9391-34d6-4a99-bd91-776ecada771e/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.658627505Z","hostname":"sock","pid":1566,"name":"oxp_276e9391-34d6-4a99-bd91-776ecada771e/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.658632045Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/276e9391-34d6-4a99-bd91-776ecada771e/crypt/debug\"), full_name: \"oxp_276e9391-34d6-4a99-bd91-776ecada771e/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.687554385Z","hostname":"sock","pid":1566,"name":"oxp_87e49e9a-4ab3-4574-b920-463506a8978e/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.687580666Z","hostname":"sock","pid":1566,"name":"oxp_87e49e9a-4ab3-4574-b920-463506a8978e/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.687584556Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/87e49e9a-4ab3-4574-b920-463506a8978e/crypt/debug\"), full_name: \"oxp_87e49e9a-4ab3-4574-b920-463506a8978e/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.716108386Z","hostname":"sock","pid":1566,"name":"oxp_d8acd544-80aa-4e82-b375-eb038fe27547/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.716136227Z","hostname":"sock","pid":1566,"name":"oxp_d8acd544-80aa-4e82-b375-eb038fe27547/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.716140047Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/d8acd544-80aa-4e82-b375-eb038fe27547/crypt/zone\"), full_name: \"oxp_d8acd544-80aa-4e82-b375-eb038fe27547/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ce66c06a-7d54-47a6-860d-59da68080fee)","v":0,"name":"SledAgent","level":50,"time":"2025-01-15T02:23:27.745349845Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.746017923Z","hostname":"sock","pid":1566,"name":"oxp_87e49e9a-4ab3-4574-b920-463506a8978e/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.746028043Z","hostname":"sock","pid":1566,"name":"oxp_87e49e9a-4ab3-4574-b920-463506a8978e/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.746043483Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/87e49e9a-4ab3-4574-b920-463506a8978e/crypt/zone\"), full_name: \"oxp_87e49e9a-4ab3-4574-b920-463506a8978e/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.775008805Z","hostname":"sock","pid":1566,"name":"oxp_27bb8d18-eb2e-4bb2-b35c-c031637c236f/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.775040436Z","hostname":"sock","pid":1566,"name":"oxp_27bb8d18-eb2e-4bb2-b35c-c031637c236f/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.775045426Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_27bb8d18-eb2e-4bb2-b35c-c031637c236f/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.824193912Z","hostname":"sock","pid":1566,"name":"oxp_03ea8739-b702-4a4b-b400-0745bd6b4f96/crypt/zone/oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.824225793Z","hostname":"sock","pid":1566,"name":"oxp_03ea8739-b702-4a4b-b400-0745bd6b4f96/crypt/zone/oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.824229873Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/03ea8739-b702-4a4b-b400-0745bd6b4f96/crypt/zone/oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f\"), full_name: \"oxp_03ea8739-b702-4a4b-b400-0745bd6b4f96/crypt/zone/oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.882426744Z","hostname":"sock","pid":1566,"name":"oxp_27bb8d18-eb2e-4bb2-b35c-c031637c236f/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.882458335Z","hostname":"sock","pid":1566,"name":"oxp_27bb8d18-eb2e-4bb2-b35c-c031637c236f/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.882462485Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/27bb8d18-eb2e-4bb2-b35c-c031637c236f/crypt/debug\"), full_name: \"oxp_27bb8d18-eb2e-4bb2-b35c-c031637c236f/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.910919303Z","hostname":"sock","pid":1566,"name":"oxp_57cb94f0-4114-4bd3-970a-a0245b825f9d/crypt/zone/oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.910950644Z","hostname":"sock","pid":1566,"name":"oxp_57cb94f0-4114-4bd3-970a-a0245b825f9d/crypt/zone/oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.910954704Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/57cb94f0-4114-4bd3-970a-a0245b825f9d/crypt/zone/oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\"), full_name: \"oxp_57cb94f0-4114-4bd3-970a-a0245b825f9d/crypt/zone/oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.968720952Z","hostname":"sock","pid":1566,"name":"oxp_276e9391-34d6-4a99-bd91-776ecada771e/crypt/zone/oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.968753063Z","hostname":"sock","pid":1566,"name":"oxp_276e9391-34d6-4a99-bd91-776ecada771e/crypt/zone/oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:27.968757273Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/276e9391-34d6-4a99-bd91-776ecada771e/crypt/zone/oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd\"), full_name: \"oxp_276e9391-34d6-4a99-bd91-776ecada771e/crypt/zone/oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.028623319Z","hostname":"sock","pid":1566,"name":"oxp_57cb94f0-4114-4bd3-970a-a0245b825f9d/crypt/zone/oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.028658389Z","hostname":"sock","pid":1566,"name":"oxp_57cb94f0-4114-4bd3-970a-a0245b825f9d/crypt/zone/oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.02866284Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/57cb94f0-4114-4bd3-970a-a0245b825f9d/crypt/zone/oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402\"), full_name: \"oxp_57cb94f0-4114-4bd3-970a-a0245b825f9d/crypt/zone/oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.089514331Z","hostname":"sock","pid":1566,"name":"oxp_27bb8d18-eb2e-4bb2-b35c-c031637c236f/crypt/zone/oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.089559563Z","hostname":"sock","pid":1566,"name":"oxp_27bb8d18-eb2e-4bb2-b35c-c031637c236f/crypt/zone/oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.089563983Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/27bb8d18-eb2e-4bb2-b35c-c031637c236f/crypt/zone/oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\"), full_name: \"oxp_27bb8d18-eb2e-4bb2-b35c-c031637c236f/crypt/zone/oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.151078853Z","hostname":"sock","pid":1566,"name":"oxp_d8acd544-80aa-4e82-b375-eb038fe27547/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.151109904Z","hostname":"sock","pid":1566,"name":"oxp_d8acd544-80aa-4e82-b375-eb038fe27547/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.151114184Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_d8acd544-80aa-4e82-b375-eb038fe27547/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.199280624Z","hostname":"sock","pid":1566,"name":"oxp_57cb94f0-4114-4bd3-970a-a0245b825f9d/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.199312234Z","hostname":"sock","pid":1566,"name":"oxp_57cb94f0-4114-4bd3-970a-a0245b825f9d/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.199316345Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/57cb94f0-4114-4bd3-970a-a0245b825f9d/crypt/zone\"), full_name: \"oxp_57cb94f0-4114-4bd3-970a-a0245b825f9d/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.230477745Z","hostname":"sock","pid":1566,"name":"oxp_27bb8d18-eb2e-4bb2-b35c-c031637c236f/crypt/zone/oxz_internal_dns_a4d0b75d-0c67-4224-af19-0510665c8775","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.230511756Z","hostname":"sock","pid":1566,"name":"oxp_27bb8d18-eb2e-4bb2-b35c-c031637c236f/crypt/zone/oxz_internal_dns_a4d0b75d-0c67-4224-af19-0510665c8775","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.230515796Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/27bb8d18-eb2e-4bb2-b35c-c031637c236f/crypt/zone/oxz_internal_dns_a4d0b75d-0c67-4224-af19-0510665c8775\"), full_name: \"oxp_27bb8d18-eb2e-4bb2-b35c-c031637c236f/crypt/zone/oxz_internal_dns_a4d0b75d-0c67-4224-af19-0510665c8775\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.289084997Z","hostname":"sock","pid":1566,"name":"oxp_03ea8739-b702-4a4b-b400-0745bd6b4f96/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.289127918Z","hostname":"sock","pid":1566,"name":"oxp_03ea8739-b702-4a4b-b400-0745bd6b4f96/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.289132988Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/03ea8739-b702-4a4b-b400-0745bd6b4f96/crypt/zone\"), full_name: \"oxp_03ea8739-b702-4a4b-b400-0745bd6b4f96/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.31957457Z","hostname":"sock","pid":1566,"name":"oxp_56c8079e-8d4e-4506-b6ff-2f0a5efab320/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.319612811Z","hostname":"sock","pid":1566,"name":"oxp_56c8079e-8d4e-4506-b6ff-2f0a5efab320/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.319617351Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_56c8079e-8d4e-4506-b6ff-2f0a5efab320/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.366489725Z","hostname":"sock","pid":1566,"name":"oxp_27bb8d18-eb2e-4bb2-b35c-c031637c236f/crypt/zone/oxz_cockroachdb_a6c98ed9-62e0-43ae-9066-c1d4ee8e0596","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.366528477Z","hostname":"sock","pid":1566,"name":"oxp_27bb8d18-eb2e-4bb2-b35c-c031637c236f/crypt/zone/oxz_cockroachdb_a6c98ed9-62e0-43ae-9066-c1d4ee8e0596","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.366533137Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/27bb8d18-eb2e-4bb2-b35c-c031637c236f/crypt/zone/oxz_cockroachdb_a6c98ed9-62e0-43ae-9066-c1d4ee8e0596\"), full_name: \"oxp_27bb8d18-eb2e-4bb2-b35c-c031637c236f/crypt/zone/oxz_cockroachdb_a6c98ed9-62e0-43ae-9066-c1d4ee8e0596\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.425630781Z","hostname":"sock","pid":1566,"name":"oxp_03ea8739-b702-4a4b-b400-0745bd6b4f96/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.425668033Z","hostname":"sock","pid":1566,"name":"oxp_03ea8739-b702-4a4b-b400-0745bd6b4f96/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.425684763Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/03ea8739-b702-4a4b-b400-0745bd6b4f96/crypt/debug\"), full_name: \"oxp_03ea8739-b702-4a4b-b400-0745bd6b4f96/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.453921265Z","hostname":"sock","pid":1566,"name":"oxp_276e9391-34d6-4a99-bd91-776ecada771e/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.453959306Z","hostname":"sock","pid":1566,"name":"oxp_276e9391-34d6-4a99-bd91-776ecada771e/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.453963586Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_276e9391-34d6-4a99-bd91-776ecada771e/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.500969734Z","hostname":"sock","pid":1566,"name":"oxp_56c8079e-8d4e-4506-b6ff-2f0a5efab320/crypt/zone/oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.501008035Z","hostname":"sock","pid":1566,"name":"oxp_56c8079e-8d4e-4506-b6ff-2f0a5efab320/crypt/zone/oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.501012566Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/56c8079e-8d4e-4506-b6ff-2f0a5efab320/crypt/zone/oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d\"), full_name: \"oxp_56c8079e-8d4e-4506-b6ff-2f0a5efab320/crypt/zone/oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.561538959Z","hostname":"sock","pid":1566,"name":"oxp_57cb94f0-4114-4bd3-970a-a0245b825f9d/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.56157453Z","hostname":"sock","pid":1566,"name":"oxp_57cb94f0-4114-4bd3-970a-a0245b825f9d/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.56157918Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_57cb94f0-4114-4bd3-970a-a0245b825f9d/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.611777235Z","hostname":"sock","pid":1566,"name":"oxp_57cb94f0-4114-4bd3-970a-a0245b825f9d/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.611820236Z","hostname":"sock","pid":1566,"name":"oxp_57cb94f0-4114-4bd3-970a-a0245b825f9d/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.611824506Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_57cb94f0-4114-4bd3-970a-a0245b825f9d/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.660028137Z","hostname":"sock","pid":1566,"name":"oxp_56c8079e-8d4e-4506-b6ff-2f0a5efab320/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.660065107Z","hostname":"sock","pid":1566,"name":"oxp_56c8079e-8d4e-4506-b6ff-2f0a5efab320/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.660069308Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/56c8079e-8d4e-4506-b6ff-2f0a5efab320/crypt/zone\"), full_name: \"oxp_56c8079e-8d4e-4506-b6ff-2f0a5efab320/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.689235685Z","hostname":"sock","pid":1566,"name":"oxp_b471fd39-9806-47fa-8ddb-0ca2f42aaf5d/crypt/zone/oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.689271566Z","hostname":"sock","pid":1566,"name":"oxp_b471fd39-9806-47fa-8ddb-0ca2f42aaf5d/crypt/zone/oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.689276546Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/b471fd39-9806-47fa-8ddb-0ca2f42aaf5d/crypt/zone/oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a\"), full_name: \"oxp_b471fd39-9806-47fa-8ddb-0ca2f42aaf5d/crypt/zone/oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ce66c06a-7d54-47a6-860d-59da68080fee)","v":0,"name":"SledAgent","level":50,"time":"2025-01-15T02:23:28.747436675Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.748527315Z","hostname":"sock","pid":1566,"name":"oxp_57cb94f0-4114-4bd3-970a-a0245b825f9d/crypt/clickhouse","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.748534995Z","hostname":"sock","pid":1566,"name":"oxp_57cb94f0-4114-4bd3-970a-a0245b825f9d/crypt/clickhouse","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.748551675Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_57cb94f0-4114-4bd3-970a-a0245b825f9d/crypt/clickhouse\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.799527331Z","hostname":"sock","pid":1566,"name":"oxp_03ea8739-b702-4a4b-b400-0745bd6b4f96/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.799561192Z","hostname":"sock","pid":1566,"name":"oxp_03ea8739-b702-4a4b-b400-0745bd6b4f96/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.799565552Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_03ea8739-b702-4a4b-b400-0745bd6b4f96/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.84950613Z","hostname":"sock","pid":1566,"name":"oxp_276e9391-34d6-4a99-bd91-776ecada771e/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.84953948Z","hostname":"sock","pid":1566,"name":"oxp_276e9391-34d6-4a99-bd91-776ecada771e/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.8495438Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/276e9391-34d6-4a99-bd91-776ecada771e/crypt/zone\"), full_name: \"oxp_276e9391-34d6-4a99-bd91-776ecada771e/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.87844454Z","hostname":"sock","pid":1566,"name":"oxp_87e49e9a-4ab3-4574-b920-463506a8978e/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.878476211Z","hostname":"sock","pid":1566,"name":"oxp_87e49e9a-4ab3-4574-b920-463506a8978e/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.878480881Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_87e49e9a-4ab3-4574-b920-463506a8978e/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.928922422Z","hostname":"sock","pid":1566,"name":"oxp_57cb94f0-4114-4bd3-970a-a0245b825f9d/crypt/zone/oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.928961704Z","hostname":"sock","pid":1566,"name":"oxp_57cb94f0-4114-4bd3-970a-a0245b825f9d/crypt/zone/oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.928966134Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/57cb94f0-4114-4bd3-970a-a0245b825f9d/crypt/zone/oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811\"), full_name: \"oxp_57cb94f0-4114-4bd3-970a-a0245b825f9d/crypt/zone/oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.99035146Z","hostname":"sock","pid":1566,"name":"oxp_27bb8d18-eb2e-4bb2-b35c-c031637c236f/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.990387441Z","hostname":"sock","pid":1566,"name":"oxp_27bb8d18-eb2e-4bb2-b35c-c031637c236f/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:28.990391911Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_27bb8d18-eb2e-4bb2-b35c-c031637c236f/crypt/internal_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.041930282Z","hostname":"sock","pid":1566,"name":"oxp_87e49e9a-4ab3-4574-b920-463506a8978e/crypt/zone/oxz_internal_dns_44ac331a-da63-47e3-a636-214c3ca86b22","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.041967123Z","hostname":"sock","pid":1566,"name":"oxp_87e49e9a-4ab3-4574-b920-463506a8978e/crypt/zone/oxz_internal_dns_44ac331a-da63-47e3-a636-214c3ca86b22","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.041971913Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/87e49e9a-4ab3-4574-b920-463506a8978e/crypt/zone/oxz_internal_dns_44ac331a-da63-47e3-a636-214c3ca86b22\"), full_name: \"oxp_87e49e9a-4ab3-4574-b920-463506a8978e/crypt/zone/oxz_internal_dns_44ac331a-da63-47e3-a636-214c3ca86b22\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.102236869Z","hostname":"sock","pid":1566,"name":"oxp_0aaa1466-164a-4540-ae41-ab17e9fc6bc8/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.10227237Z","hostname":"sock","pid":1566,"name":"oxp_0aaa1466-164a-4540-ae41-ab17e9fc6bc8/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.1022769Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/0aaa1466-164a-4540-ae41-ab17e9fc6bc8/crypt/zone\"), full_name: \"oxp_0aaa1466-164a-4540-ae41-ab17e9fc6bc8/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.131413546Z","hostname":"sock","pid":1566,"name":"oxp_03ea8739-b702-4a4b-b400-0745bd6b4f96/crypt/zone/oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.131445007Z","hostname":"sock","pid":1566,"name":"oxp_03ea8739-b702-4a4b-b400-0745bd6b4f96/crypt/zone/oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.131449017Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/03ea8739-b702-4a4b-b400-0745bd6b4f96/crypt/zone/oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b\"), full_name: \"oxp_03ea8739-b702-4a4b-b400-0745bd6b4f96/crypt/zone/oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.191682443Z","hostname":"sock","pid":1566,"name":"oxp_03ea8739-b702-4a4b-b400-0745bd6b4f96/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.191717493Z","hostname":"sock","pid":1566,"name":"oxp_03ea8739-b702-4a4b-b400-0745bd6b4f96/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.191722113Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_03ea8739-b702-4a4b-b400-0745bd6b4f96/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.239735789Z","hostname":"sock","pid":1566,"name":"oxp_b471fd39-9806-47fa-8ddb-0ca2f42aaf5d/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.23976747Z","hostname":"sock","pid":1566,"name":"oxp_b471fd39-9806-47fa-8ddb-0ca2f42aaf5d/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.239772Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/b471fd39-9806-47fa-8ddb-0ca2f42aaf5d/crypt/zone\"), full_name: \"oxp_b471fd39-9806-47fa-8ddb-0ca2f42aaf5d/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.26866385Z","hostname":"sock","pid":1566,"name":"oxp_b471fd39-9806-47fa-8ddb-0ca2f42aaf5d/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.268700161Z","hostname":"sock","pid":1566,"name":"oxp_b471fd39-9806-47fa-8ddb-0ca2f42aaf5d/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.268709871Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/b471fd39-9806-47fa-8ddb-0ca2f42aaf5d/crypt/debug\"), full_name: \"oxp_b471fd39-9806-47fa-8ddb-0ca2f42aaf5d/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.299869092Z","hostname":"sock","pid":1566,"name":"oxp_d8acd544-80aa-4e82-b375-eb038fe27547/crypt/zone/oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.299904263Z","hostname":"sock","pid":1566,"name":"oxp_d8acd544-80aa-4e82-b375-eb038fe27547/crypt/zone/oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.299908403Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/d8acd544-80aa-4e82-b375-eb038fe27547/crypt/zone/oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344\"), full_name: \"oxp_d8acd544-80aa-4e82-b375-eb038fe27547/crypt/zone/oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.360454997Z","hostname":"sock","pid":1566,"name":"oxp_87e49e9a-4ab3-4574-b920-463506a8978e/crypt/zone/oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.360489378Z","hostname":"sock","pid":1566,"name":"oxp_87e49e9a-4ab3-4574-b920-463506a8978e/crypt/zone/oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.360493498Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/87e49e9a-4ab3-4574-b920-463506a8978e/crypt/zone/oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e\"), full_name: \"oxp_87e49e9a-4ab3-4574-b920-463506a8978e/crypt/zone/oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.423325813Z","hostname":"sock","pid":1566,"name":"oxp_276e9391-34d6-4a99-bd91-776ecada771e/crypt/zone/oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.423360504Z","hostname":"sock","pid":1566,"name":"oxp_276e9391-34d6-4a99-bd91-776ecada771e/crypt/zone/oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.423364624Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/276e9391-34d6-4a99-bd91-776ecada771e/crypt/zone/oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\"), full_name: \"oxp_276e9391-34d6-4a99-bd91-776ecada771e/crypt/zone/oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.483444605Z","hostname":"sock","pid":1566,"name":"oxp_b471fd39-9806-47fa-8ddb-0ca2f42aaf5d/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.483478586Z","hostname":"sock","pid":1566,"name":"oxp_b471fd39-9806-47fa-8ddb-0ca2f42aaf5d/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.483482757Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_b471fd39-9806-47fa-8ddb-0ca2f42aaf5d/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.533533367Z","hostname":"sock","pid":1566,"name":"oxp_57cb94f0-4114-4bd3-970a-a0245b825f9d/crypt/zone/oxz_cockroachdb_77bd825f-cbb2-4b31-ba9f-d69e9e4cff59","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.533573428Z","hostname":"sock","pid":1566,"name":"oxp_57cb94f0-4114-4bd3-970a-a0245b825f9d/crypt/zone/oxz_cockroachdb_77bd825f-cbb2-4b31-ba9f-d69e9e4cff59","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.533577428Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/57cb94f0-4114-4bd3-970a-a0245b825f9d/crypt/zone/oxz_cockroachdb_77bd825f-cbb2-4b31-ba9f-d69e9e4cff59\"), full_name: \"oxp_57cb94f0-4114-4bd3-970a-a0245b825f9d/crypt/zone/oxz_cockroachdb_77bd825f-cbb2-4b31-ba9f-d69e9e4cff59\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.592586271Z","hostname":"sock","pid":1566,"name":"oxp_b471fd39-9806-47fa-8ddb-0ca2f42aaf5d/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.592620262Z","hostname":"sock","pid":1566,"name":"oxp_b471fd39-9806-47fa-8ddb-0ca2f42aaf5d/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.592624812Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_b471fd39-9806-47fa-8ddb-0ca2f42aaf5d/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.641748867Z","hostname":"sock","pid":1566,"name":"oxp_276e9391-34d6-4a99-bd91-776ecada771e/crypt/zone/oxz_ntp_fd5ec85c-1d4a-43a3-bd64-3bf287ecded3","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.641787268Z","hostname":"sock","pid":1566,"name":"oxp_276e9391-34d6-4a99-bd91-776ecada771e/crypt/zone/oxz_ntp_fd5ec85c-1d4a-43a3-bd64-3bf287ecded3","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.641791758Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/276e9391-34d6-4a99-bd91-776ecada771e/crypt/zone/oxz_ntp_fd5ec85c-1d4a-43a3-bd64-3bf287ecded3\"), full_name: \"oxp_276e9391-34d6-4a99-bd91-776ecada771e/crypt/zone/oxz_ntp_fd5ec85c-1d4a-43a3-bd64-3bf287ecded3\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.702759764Z","hostname":"sock","pid":1566,"name":"oxp_b471fd39-9806-47fa-8ddb-0ca2f42aaf5d/crypt/zone/oxz_cockroachdb_82a65d0c-f391-4b1f-9c11-e3a397904f10","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.702794685Z","hostname":"sock","pid":1566,"name":"oxp_b471fd39-9806-47fa-8ddb-0ca2f42aaf5d/crypt/zone/oxz_cockroachdb_82a65d0c-f391-4b1f-9c11-e3a397904f10","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.702798995Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/b471fd39-9806-47fa-8ddb-0ca2f42aaf5d/crypt/zone/oxz_cockroachdb_82a65d0c-f391-4b1f-9c11-e3a397904f10\"), full_name: \"oxp_b471fd39-9806-47fa-8ddb-0ca2f42aaf5d/crypt/zone/oxz_cockroachdb_82a65d0c-f391-4b1f-9c11-e3a397904f10\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ce66c06a-7d54-47a6-860d-59da68080fee)","v":0,"name":"SledAgent","level":50,"time":"2025-01-15T02:23:29.749225968Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.761987572Z","hostname":"sock","pid":1566,"name":"oxp_87e49e9a-4ab3-4574-b920-463506a8978e/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.762001232Z","hostname":"sock","pid":1566,"name":"oxp_87e49e9a-4ab3-4574-b920-463506a8978e/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.762009813Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_87e49e9a-4ab3-4574-b920-463506a8978e/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.810326486Z","hostname":"sock","pid":1566,"name":"oxp_56c8079e-8d4e-4506-b6ff-2f0a5efab320/crypt/zone/oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.810379328Z","hostname":"sock","pid":1566,"name":"oxp_56c8079e-8d4e-4506-b6ff-2f0a5efab320/crypt/zone/oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.810386338Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/56c8079e-8d4e-4506-b6ff-2f0a5efab320/crypt/zone/oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7\"), full_name: \"oxp_56c8079e-8d4e-4506-b6ff-2f0a5efab320/crypt/zone/oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.872692849Z","hostname":"sock","pid":1566,"name":"oxp_57cb94f0-4114-4bd3-970a-a0245b825f9d/crypt/zone/oxz_internal_dns_0e94e881-2bab-4d68-8315-741d3cd1814a","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.87273133Z","hostname":"sock","pid":1566,"name":"oxp_57cb94f0-4114-4bd3-970a-a0245b825f9d/crypt/zone/oxz_internal_dns_0e94e881-2bab-4d68-8315-741d3cd1814a","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.87273573Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/57cb94f0-4114-4bd3-970a-a0245b825f9d/crypt/zone/oxz_internal_dns_0e94e881-2bab-4d68-8315-741d3cd1814a\"), full_name: \"oxp_57cb94f0-4114-4bd3-970a-a0245b825f9d/crypt/zone/oxz_internal_dns_0e94e881-2bab-4d68-8315-741d3cd1814a\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.947146578Z","hostname":"sock","pid":1566,"name":"oxp_57cb94f0-4114-4bd3-970a-a0245b825f9d/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"Old properties missing UUID","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.947188799Z","hostname":"sock","pid":1566,"name":"oxp_57cb94f0-4114-4bd3-970a-a0245b825f9d/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:998"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.94719384Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/57cb94f0-4114-4bd3-970a-a0245b825f9d/crypt/debug\"), full_name: \"oxp_57cb94f0-4114-4bd3-970a-a0245b825f9d/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.976750117Z","hostname":"sock","pid":1566,"name":"oxp_57cb94f0-4114-4bd3-970a-a0245b825f9d/crypt/external_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1039"} {"msg":"This dataset did not exist","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.976788508Z","hostname":"sock","pid":1566,"name":"oxp_57cb94f0-4114-4bd3-970a-a0245b825f9d/crypt/external_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:993"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:29.976793778Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1431","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_57cb94f0-4114-4bd3-970a-a0245b825f9d/crypt/external_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Writing ledger to /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/.omicron-datasets.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:30.028658247Z","hostname":"sock","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/.omicron-datasets.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:30.028987156Z","hostname":"sock","pid":1566,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"datasets ensure: Updated storage","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:30.029290774Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:861"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:30.029394027Z","hostname":"sock","pid":1566,"uri":"/datasets","method":"PUT","req_id":"4180bd51-a5c3-4c8c-bd23-a8f5b4a69043","remote_addr":"[fd00:1122:3344:101::1]:63500","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":3239766,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: 0e94e881-2bab-4d68-8315-741d3cd1814a (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a4d0b75d-0c67-4224-af19-0510665c8775 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 44ac331a-da63-47e3-a636-214c3ca86b22 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:30.061100953Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:30.061240347Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:62970"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:30.061529374Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:30.061546105Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:30.06988251Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:30.069921271Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:30.078742109Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:30.07877764Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:30.087472424Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"No ledger in /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:30.087515476Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:30.087524616Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:30.087573777Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:30.130856035Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:30.130899336Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:30.130903126Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_0e94e881-2bab-4d68-8315-741d3cd1814a","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:30.185051337Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_a4d0b75d-0c67-4224-af19-0510665c8775","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:30.189056726Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_44ac331a-da63-47e3-a636-214c3ca86b22","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:30.193030313Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_internal_dns_0e94e881-2bab-4d68-8315-741d3cd1814a","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:30.204841262Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_internal_dns_a4d0b75d-0c67-4224-af19-0510665c8775","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:30.209461296Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_internal_dns_44ac331a-da63-47e3-a636-214c3ca86b22","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:30.213786783Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ce66c06a-7d54-47a6-860d-59da68080fee)","v":0,"name":"SledAgent","level":50,"time":"2025-01-15T02:23:30.751584625Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Profile for oxz_internal_dns_a4d0b75d-0c67-4224-af19-0510665c8775:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:31.843223272Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:31.843287694Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:2::, width: 64 }"} {"msg":"Booting oxz_internal_dns_a4d0b75d-0c67-4224-af19-0510665c8775 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:31.843482759Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_a4d0b75d-0c67-4224-af19-0510665c8775","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_internal_dns_44ac331a-da63-47e3-a636-214c3ca86b22:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:32.327268873Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:32.327322304Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:3::, width: 64 }"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-15T02:23:32.544025592Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5049691774691987306"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-15T02:23:32.544119425Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"7304221604471264573"} {"msg":"Profile for oxz_internal_dns_0e94e881-2bab-4d68-8315-741d3cd1814a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:32.810271807Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_internal_dns_a4d0b75d-0c67-4224-af19-0510665c8775\") failed: Property not found. retry in 57.599778ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:32.818099308Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_a4d0b75d-0c67-4224-af19-0510665c8775","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"Booting oxz_internal_dns_44ac331a-da63-47e3-a636-214c3ca86b22 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:32.818144429Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_44ac331a-da63-47e3-a636-214c3ca86b22","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:32.820259636Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4307"} {"msg":"Ensuring there is a default route","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:32.820333478Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4317","gateway":"fd00:1122:3344:101::1"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:32.824578063Z","hostname":"sock","pid":1566,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:1::, width: 64 }"} {"msg":"Booting oxz_internal_dns_0e94e881-2bab-4d68-8315-741d3cd1814a zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:32.824643164Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_0e94e881-2bab-4d68-8315-741d3cd1814a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Default route already exists","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:32.8289281Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4331","gateway":"fd00:1122:3344:101::1"} {"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:32.828953101Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4396"} {"msg":"wait for service Some(\"oxz_internal_dns_a4d0b75d-0c67-4224-af19-0510665c8775\") failed: Property not found. retry in 51.188705ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:32.886957636Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_a4d0b75d-0c67-4224-af19-0510665c8775","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_internal_dns_a4d0b75d-0c67-4224-af19-0510665c8775\") failed: Property not found. retry in 199.793286ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:32.949212016Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_a4d0b75d-0c67-4224-af19-0510665c8775","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_internal_dns_44ac331a-da63-47e3-a636-214c3ca86b22\") failed: Property not found. retry in 59.508213ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:32.956073921Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_44ac331a-da63-47e3-a636-214c3ca86b22","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_internal_dns_0e94e881-2bab-4d68-8315-741d3cd1814a\") failed: Property not found. retry in 58.542897ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:32.966300006Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_0e94e881-2bab-4d68-8315-741d3cd1814a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"refreshed dendrite service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:32.989498252Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4462"} {"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:32.989545394Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4489"} {"msg":"wait for service Some(\"oxz_internal_dns_44ac331a-da63-47e3-a636-214c3ca86b22\") failed: Property not found. retry in 132.156072ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:33.152243534Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_44ac331a-da63-47e3-a636-214c3ca86b22","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_internal_dns_0e94e881-2bab-4d68-8315-741d3cd1814a\") failed: Property not found. retry in 95.142378ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:33.275239883Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_0e94e881-2bab-4d68-8315-741d3cd1814a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"refreshed lldpd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:33.289728494Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4502"} {"msg":"configuring MGS service","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:33.289794766Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4348"} {"msg":"wait for service Some(\"oxz_internal_dns_a4d0b75d-0c67-4224-af19-0510665c8775\") failed: Property not found. retry in 544.472498ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:33.298574063Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_a4d0b75d-0c67-4224-af19-0510665c8775","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_internal_dns_44ac331a-da63-47e3-a636-214c3ca86b22\") failed: Property not found. retry in 216.926358ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:33.342443956Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_44ac331a-da63-47e3-a636-214c3ca86b22","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_internal_dns_0e94e881-2bab-4d68-8315-741d3cd1814a\") failed: Property not found. retry in 170.550283ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:33.388856209Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_0e94e881-2bab-4d68-8315-741d3cd1814a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"refreshed MGS service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:33.445623581Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4390"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:33.445674782Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4469","rack_subnet":"fd00:1122:3344::"} {"msg":"refreshed wicketd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:33.47154444Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4480"} {"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:33.471577981Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4522"} {"msg":"refreshed mgd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:33.551013095Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4554"} {"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:33.551048366Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4560"} {"msg":"wait for service Some(\"oxz_internal_dns_0e94e881-2bab-4d68-8315-741d3cd1814a\") failed: Property not found. retry in 587.801485ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:33.570502121Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_0e94e881-2bab-4d68-8315-741d3cd1814a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_internal_dns_44ac331a-da63-47e3-a636-214c3ca86b22\") failed: Property not found. retry in 491.120053ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:33.582084483Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_44ac331a-da63-47e3-a636-214c3ca86b22","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"refreshed mg-ddm service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:33.679380328Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4601"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:33.67943194Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:370"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:33.713114179Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:420"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:33.745723989Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:717"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:33.745816521Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:762"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:33.746357516Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:765","uptime":"1"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:33.746374896Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:444","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.2.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: Some(\n None,\n ),\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n tx_eq: None,\n },\n },\n },\n}"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:33.78320454Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:33.837408333Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:33.837454374Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:33.837462554Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:33.837468645Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:33.837489445Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:33.837494915Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:33.837500265Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:33.837503675Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:33.837508866Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:33.837512916Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:33.837517146Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:33.837522186Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:33.837525576Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:33.837528796Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:33.837532116Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:33.837535466Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:33.837542067Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:33.837545397Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:33.837548677Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:33.837552707Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:33.837555957Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:33.837559217Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:33.837562447Z","hostname":"sock","pid":1566,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"wait for service Some(\"oxz_internal_dns_a4d0b75d-0c67-4224-af19-0510665c8775\") failed: Property not found. retry in 928.752197ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:33.852342886Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_a4d0b75d-0c67-4224-af19-0510665c8775","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-15T02:23:33.900638409Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"413521614313442029"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-15T02:23:33.900758972Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"17478185358689782290"} {"msg":"wait for service Some(\"oxz_internal_dns_44ac331a-da63-47e3-a636-214c3ca86b22\") failed: Property not found. retry in 860.26217ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:34.08375028Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_44ac331a-da63-47e3-a636-214c3ca86b22","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_internal_dns_0e94e881-2bab-4d68-8315-741d3cd1814a\") failed: Property not found. retry in 1.043479885s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:34.169013271Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_0e94e881-2bab-4d68-8315-741d3cd1814a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_internal_dns_a4d0b75d-0c67-4224-af19-0510665c8775\") failed: Property not found. retry in 1.47255747s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:34.790553252Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_a4d0b75d-0c67-4224-af19-0510665c8775","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_internal_dns_44ac331a-da63-47e3-a636-214c3ca86b22\") failed: Property not found. retry in 1.227063731s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:34.95388407Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_44ac331a-da63-47e3-a636-214c3ca86b22","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_internal_dns_0e94e881-2bab-4d68-8315-741d3cd1814a\") failed: Property not found. retry in 1.31793451s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:35.222405965Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_0e94e881-2bab-4d68-8315-741d3cd1814a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_internal_dns_44ac331a-da63-47e3-a636-214c3ca86b22\") failed: Property not found. retry in 1.300136233s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:36.188980808Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_44ac331a-da63-47e3-a636-214c3ca86b22","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_internal_dns_a4d0b75d-0c67-4224-af19-0510665c8775\") failed: Property not found. retry in 1.186098731s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:36.271184516Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_a4d0b75d-0c67-4224-af19-0510665c8775","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_internal_dns_0e94e881-2bab-4d68-8315-741d3cd1814a\") failed: Property not found. retry in 907.40573ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:36.548771427Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_0e94e881-2bab-4d68-8315-741d3cd1814a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_internal_dns_0e94e881-2bab-4d68-8315-741d3cd1814a\") failed: Property not found. retry in 833.846824ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:37.467046175Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_0e94e881-2bab-4d68-8315-741d3cd1814a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_internal_dns_44ac331a-da63-47e3-a636-214c3ca86b22\") failed: Property not found. retry in 757.996567ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:37.499372387Z","hostname":"sock","pid":1566,"zone":"oxz_internal_dns_44ac331a-da63-47e3-a636-214c3ca86b22","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:38.320012051Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_internal_dns_0e94e881-2bab-4d68-8315-741d3cd1814a"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:38.320047112Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_internal_dns_a4d0b75d-0c67-4224-af19-0510665c8775"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:38.320051392Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_internal_dns_44ac331a-da63-47e3-a636-214c3ca86b22"} {"msg":"Writing ledger to /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:38.320057803Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:38.320357061Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:38.320614058Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"253a2c70-a838-40b8-bbc8-4ab01f16acae","remote_addr":"[fd00:1122:3344:101::1]:62970","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8259270,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:38.322610412Z","hostname":"sock","pid":1566,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:658"} {"msg":"failed to write DNS configuration (will retry in 58.91389ms)","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:38.774211778Z","hostname":"sock","pid":1566,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:678","error_message":"Communication Error: error sending request for url (http://[fd00:1122:3344:1::1]:5353/config): client error (Connect): tcp connect error: Connection refused (os error 146): Connection refused (os error 146)"} {"msg":"failed to write DNS configuration (will retry in 176.675713ms)","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:38.834446393Z","hostname":"sock","pid":1566,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:678","error_message":"Communication Error: error sending request for url (http://[fd00:1122:3344:1::1]:5353/config): client error (Connect): tcp connect error: Connection refused (os error 146): Connection refused (os error 146)"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:39.031010027Z","hostname":"sock","pid":1566,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:658"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:39.08079132Z","hostname":"sock","pid":1566,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:658"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:39.127566852Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:694"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:39.127580193Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:39.157907121Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:39.158738614Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:39.158754754Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: 0e94e881-2bab-4d68-8315-741d3cd1814a (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a4d0b75d-0c67-4224-af19-0510665c8775 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 44ac331a-da63-47e3-a636-214c3ca86b22 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 46c4b5d7-b839-4d27-ac2d-e1f6f6eae7d3, kind: Service { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 }, name: Name(\"ntp-fd5ec85c-1d4a-43a3-bd64-3bf287ecded3\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 9])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:39.188196789Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:39.188292331Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:44911"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:39.188488786Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:39.188502437Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:39.19937001Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:39.199411921Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:39.2090252Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:39.209056061Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:39.218485616Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:39.218513227Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:39.218675351Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:39.218799834Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:39.23013494Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:39.262715679Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:39.263360307Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:39.263367257Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:39.293447049Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:447","port":"Port { inner: PortInner(PortData { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 242, 9]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:39.293456669Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:1264","zone_type":"boundary_ntp"} {"msg":"no nat entry found for: 10.151.2.169","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:39.293791808Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.169","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:39.293798018Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.2.169","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:39.294009694Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:39.294385084Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Configuring new Omicron zone: oxz_ntp_fd5ec85c-1d4a-43a3-bd64-3bf287ecded3","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:39.318652989Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_ntp_fd5ec85c-1d4a-43a3-bd64-3bf287ecded3","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:39.336368797Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Profile for oxz_ntp_fd5ec85c-1d4a-43a3-bd64-3bf287ecded3:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:39.988460223Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_ntp_fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:39.988582986Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_fd5ec85c-1d4a-43a3-bd64-3bf287ecded3","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_ntp_fd5ec85c-1d4a-43a3-bd64-3bf287ecded3\") failed: Property not found. retry in 46.366128ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:40.114513264Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_fd5ec85c-1d4a-43a3-bd64-3bf287ecded3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_ntp_fd5ec85c-1d4a-43a3-bd64-3bf287ecded3\") failed: Property not found. retry in 134.345764ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:40.168518981Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_fd5ec85c-1d4a-43a3-bd64-3bf287ecded3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_ntp_fd5ec85c-1d4a-43a3-bd64-3bf287ecded3\") failed: Property not found. retry in 104.989444ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:40.313018061Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_fd5ec85c-1d4a-43a3-bd64-3bf287ecded3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_ntp_fd5ec85c-1d4a-43a3-bd64-3bf287ecded3\") failed: Property not found. retry in 295.126774ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:40.426104882Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_fd5ec85c-1d4a-43a3-bd64-3bf287ecded3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_ntp_fd5ec85c-1d4a-43a3-bd64-3bf287ecded3\") failed: Property not found. retry in 741.12454ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:40.730070184Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_fd5ec85c-1d4a-43a3-bd64-3bf287ecded3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_ntp_fd5ec85c-1d4a-43a3-bd64-3bf287ecded3\") failed: Property not found. retry in 546.978966ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:41.480926505Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_fd5ec85c-1d4a-43a3-bd64-3bf287ecded3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_ntp_fd5ec85c-1d4a-43a3-bd64-3bf287ecded3\") failed: Property not found. retry in 1.215003944s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:42.038894312Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_fd5ec85c-1d4a-43a3-bd64-3bf287ecded3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-15T02:23:42.54653439Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5049691774691987306"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-15T02:23:42.546662754Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"7304221604471264573"} {"msg":"wait for service Some(\"oxz_ntp_fd5ec85c-1d4a-43a3-bd64-3bf287ecded3\") failed: Property not found. retry in 546.764008ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:43.262610662Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_fd5ec85c-1d4a-43a3-bd64-3bf287ecded3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ce66c06a-7d54-47a6-860d-59da68080fee)","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:43.715086522Z","hostname":"sock","pid":1566,"component":"NexusNotifierTask","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/nexus.rs:527"} {"msg":"wait for service Some(\"oxz_ntp_fd5ec85c-1d4a-43a3-bd64-3bf287ecded3\") failed: Property not found. retry in 1.053599313s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:43.818157444Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_fd5ec85c-1d4a-43a3-bd64-3bf287ecded3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-15T02:23:43.90252297Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"413521614313442029"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-15T02:23:43.902652324Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"17478185358689782290"} {"msg":"wait for service Some(\"oxz_ntp_fd5ec85c-1d4a-43a3-bd64-3bf287ecded3\") failed: Property not found. retry in 1.275022153s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:44.883150541Z","hostname":"sock","pid":1566,"zone":"oxz_ntp_fd5ec85c-1d4a-43a3-bd64-3bf287ecded3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:46.178187626Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_ntp_fd5ec85c-1d4a-43a3-bd64-3bf287ecded3"} {"msg":"Writing ledger to /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:46.178235288Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:46.178658479Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:46.179024569Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"f3a1197e-489b-4c5a-aae9-64c4bda9b33a","remote_addr":"[fd00:1122:3344:101::1]:44911","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":6990666,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:46.181516776Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:735"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:46.212585364Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:46.212698538Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:49230"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_fd5ec85c-1d4a-43a3-bd64-3bf287ecded3': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2025-01-15T02:23:46.220805236Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3915"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:46.220916589Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"83fa9dd9-ec3c-4aab-812d-05b56874b93c","remote_addr":"[fd00:1122:3344:101::1]:49230","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8088,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:46.222587604Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:46.347388862Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:46.347516005Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:40638"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-15T02:23:47.48948166Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"3930529367751985057"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-15T02:23:48.283443715Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"2741447075642311501"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2025-01-15T02:23:48.322740055Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12541906727051864147"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_fd5ec85c-1d4a-43a3-bd64-3bf287ecded3': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2025-01-15T02:23:50.355464696Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3915"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:50.355535598Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"353d3b24-2671-47c0-bc3d-d48dd992c45f","remote_addr":"[fd00:1122:3344:101::1]:40638","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4007966,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:50.357688486Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:50.516923093Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:50.517081677Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:51302"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:50.525966717Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"d741e961-03de-4107-b2bd-732b8959548b","remote_addr":"[fd00:1122:3344:101::1]:51302","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8830,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:50.527831027Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:50.527837127Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:50.695220094Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:50.695336307Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:55008"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:50.703100987Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"b120cb4d-6990-4357-98ed-dddc78c71d43","remote_addr":"[fd00:1122:3344:101::1]:55008","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7683,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:50.70545671Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:50.70546304Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:50.945001264Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:50.945106927Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:37700"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:50.952500077Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"a853be67-9e30-4c52-8408-ab5044600e86","remote_addr":"[fd00:1122:3344:101::1]:37700","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7327,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:50.95449181Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:50.95449766Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:51.195634447Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:51.195803802Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:37786"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:51.204151727Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"501fb483-5c14-46ca-a6cf-d9f69ac21be4","remote_addr":"[fd00:1122:3344:101::1]:37786","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8297,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:51.206691085Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:51.206697556Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:51.440667179Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:51.440785962Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:64926"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:51.448397848Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"52055a45-dd79-4a9b-94b9-38e479bf41aa","remote_addr":"[fd00:1122:3344:101::1]:64926","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7570,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:51.450453083Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:51.450461074Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:765","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:52.044140091Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:715"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:52.044262204Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:54266"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:52.06339366Z","hostname":"sock","pid":1566,"uri":"/timesync","method":"GET","req_id":"d0120f1d-d60c-45d8-b954-2c7dbaf312b6","remote_addr":"[fd00:1122:3344:101::1]:54266","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":19059,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 395716986, ip_addr: 23.150.41.122, stratum: 5, ref_time: 1736907831.2973514, correction: -0.0 }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:52.066153445Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:743"} {"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:52.066160045Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1392"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 0e94e881-2bab-4d68-8315-741d3cd1814a (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a4d0b75d-0c67-4224-af19-0510665c8775 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 44ac331a-da63-47e3-a636-214c3ca86b22 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 46c4b5d7-b839-4d27-ac2d-e1f6f6eae7d3, kind: Service { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 }, name: Name(\"ntp-fd5ec85c-1d4a-43a3-bd64-3bf287ecded3\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 9])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 77bd825f-cbb2-4b31-ba9f-d69e9e4cff59 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: a6c98ed9-62e0-43ae-9066-c1d4ee8e0596 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 788bf241-055b-45bf-8d67-e865c6324e46 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 82a65d0c-f391-4b1f-9c11-e3a397904f10 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 03e9536c-5029-447f-944a-4daff2909c7f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:52.027456932Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:52.027544734Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:46046"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:52.027735829Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:52.027742169Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:52.03775985Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:52.03779088Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:52.046849925Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:52.046889816Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:52.054621734Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:52.054651985Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:52.063655168Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:52.063686169Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:52.072024404Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:52.072053985Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:52.08225732Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:52.082290341Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:52.090654547Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:52.090687588Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:52.098308113Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:52.098338624Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:52.098523789Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:52.098640662Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:52.181311813Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:52.181360354Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:52.181366485Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:52.181374085Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:52.181379945Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_77bd825f-cbb2-4b31-ba9f-d69e9e4cff59","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:52.270267373Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:52.274165509Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_788bf241-055b-45bf-8d67-e865c6324e46","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:52.278437564Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_a6c98ed9-62e0-43ae-9066-c1d4ee8e0596","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:52.282756221Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_82a65d0c-f391-4b1f-9c11-e3a397904f10","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:52.287150169Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_cockroachdb_77bd825f-cbb2-4b31-ba9f-d69e9e4cff59","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:52.291719472Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:52.296593963Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_cockroachdb_788bf241-055b-45bf-8d67-e865c6324e46","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:52.301133466Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_cockroachdb_a6c98ed9-62e0-43ae-9066-c1d4ee8e0596","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:52.305339239Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_cockroachdb_82a65d0c-f391-4b1f-9c11-e3a397904f10","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:52.308927146Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Profile for oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:55.62105663Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:55.621175845Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_a6c98ed9-62e0-43ae-9066-c1d4ee8e0596:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:55.65037178Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_a6c98ed9-62e0-43ae-9066-c1d4ee8e0596 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:55.650609883Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_a6c98ed9-62e0-43ae-9066-c1d4ee8e0596","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_82a65d0c-f391-4b1f-9c11-e3a397904f10:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:55.691657031Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_82a65d0c-f391-4b1f-9c11-e3a397904f10 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:55.691793488Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_82a65d0c-f391-4b1f-9c11-e3a397904f10","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f\") failed: Property not found. retry in 47.900354ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:55.812601377Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_a6c98ed9-62e0-43ae-9066-c1d4ee8e0596\") failed: Property not found. retry in 63.995692ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:55.842235244Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_a6c98ed9-62e0-43ae-9066-c1d4ee8e0596","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"Profile for oxz_cockroachdb_77bd825f-cbb2-4b31-ba9f-d69e9e4cff59:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:55.842637873Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_77bd825f-cbb2-4b31-ba9f-d69e9e4cff59 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:55.842798265Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_77bd825f-cbb2-4b31-ba9f-d69e9e4cff59","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f\") failed: Property not found. retry in 79.415685ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:55.870064492Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_82a65d0c-f391-4b1f-9c11-e3a397904f10\") failed: Property not found. retry in 61.948445ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:55.87879619Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_82a65d0c-f391-4b1f-9c11-e3a397904f10","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_a6c98ed9-62e0-43ae-9066-c1d4ee8e0596\") failed: Property not found. retry in 86.755299ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:55.914530581Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_a6c98ed9-62e0-43ae-9066-c1d4ee8e0596","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_82a65d0c-f391-4b1f-9c11-e3a397904f10\") failed: Property not found. retry in 69.298837ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:55.951048716Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_82a65d0c-f391-4b1f-9c11-e3a397904f10","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f\") failed: Property not found. retry in 100.540225ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:55.959581439Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_77bd825f-cbb2-4b31-ba9f-d69e9e4cff59\") failed: Property not found. retry in 46.432639ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:56.008067999Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_77bd825f-cbb2-4b31-ba9f-d69e9e4cff59","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_a6c98ed9-62e0-43ae-9066-c1d4ee8e0596\") failed: Property not found. retry in 241.263031ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:56.017056488Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_a6c98ed9-62e0-43ae-9066-c1d4ee8e0596","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"Profile for oxz_cockroachdb_788bf241-055b-45bf-8d67-e865c6324e46:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:56.017617709Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_788bf241-055b-45bf-8d67-e865c6324e46 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:23:56.017855522Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_788bf241-055b-45bf-8d67-e865c6324e46","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_cockroachdb_82a65d0c-f391-4b1f-9c11-e3a397904f10\") failed: Property not found. retry in 216.996999ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:56.031100857Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_82a65d0c-f391-4b1f-9c11-e3a397904f10","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_77bd825f-cbb2-4b31-ba9f-d69e9e4cff59\") failed: Property not found. retry in 131.476253ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:56.064558517Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_77bd825f-cbb2-4b31-ba9f-d69e9e4cff59","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f\") failed: Property not found. retry in 222.4815ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:56.074716686Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_788bf241-055b-45bf-8d67-e865c6324e46\") failed: Property not found. retry in 54.744809ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:56.180882699Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_788bf241-055b-45bf-8d67-e865c6324e46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_77bd825f-cbb2-4b31-ba9f-d69e9e4cff59\") failed: Property not found. retry in 219.905854ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:56.208953812Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_77bd825f-cbb2-4b31-ba9f-d69e9e4cff59","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_788bf241-055b-45bf-8d67-e865c6324e46\") failed: Property not found. retry in 121.262959ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:56.24595967Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_788bf241-055b-45bf-8d67-e865c6324e46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_82a65d0c-f391-4b1f-9c11-e3a397904f10\") failed: Property not found. retry in 541.453188ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:56.261261701Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_82a65d0c-f391-4b1f-9c11-e3a397904f10","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_a6c98ed9-62e0-43ae-9066-c1d4ee8e0596\") failed: Property not found. retry in 211.360933ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:56.275582608Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_a6c98ed9-62e0-43ae-9066-c1d4ee8e0596","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f\") failed: Property not found. retry in 599.204279ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:56.309675908Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_788bf241-055b-45bf-8d67-e865c6324e46\") failed: Property not found. retry in 231.636973ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:56.37975959Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_788bf241-055b-45bf-8d67-e865c6324e46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_77bd825f-cbb2-4b31-ba9f-d69e9e4cff59\") failed: Property not found. retry in 564.865051ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:56.439869053Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_77bd825f-cbb2-4b31-ba9f-d69e9e4cff59","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_a6c98ed9-62e0-43ae-9066-c1d4ee8e0596\") failed: Property not found. retry in 1.161331232s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:56.497637663Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_a6c98ed9-62e0-43ae-9066-c1d4ee8e0596","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_788bf241-055b-45bf-8d67-e865c6324e46\") failed: Property not found. retry in 334.876603ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:56.622135008Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_788bf241-055b-45bf-8d67-e865c6324e46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:56.734288679Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"284.111793ms"} {"msg":"wait for service Some(\"oxz_cockroachdb_82a65d0c-f391-4b1f-9c11-e3a397904f10\") failed: Property not found. retry in 615.376862ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:56.814668199Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_82a65d0c-f391-4b1f-9c11-e3a397904f10","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f\") failed: Property not found. retry in 1.389253686s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:56.922330055Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_788bf241-055b-45bf-8d67-e865c6324e46\") failed: Property not found. retry in 766.554685ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:56.967770024Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_788bf241-055b-45bf-8d67-e865c6324e46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_77bd825f-cbb2-4b31-ba9f-d69e9e4cff59\") failed: Property not found. retry in 1.0528704s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:57.01588714Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_77bd825f-cbb2-4b31-ba9f-d69e9e4cff59","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_82a65d0c-f391-4b1f-9c11-e3a397904f10\") failed: Property not found. retry in 1.093613244s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:57.460896107Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_82a65d0c-f391-4b1f-9c11-e3a397904f10","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_a6c98ed9-62e0-43ae-9066-c1d4ee8e0596\") failed: Property not found. retry in 537.681876ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:57.674813507Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_a6c98ed9-62e0-43ae-9066-c1d4ee8e0596","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_788bf241-055b-45bf-8d67-e865c6324e46\") failed: Property not found. retry in 1.06791714s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:57.750392008Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_788bf241-055b-45bf-8d67-e865c6324e46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_77bd825f-cbb2-4b31-ba9f-d69e9e4cff59\") failed: Property not found. retry in 514.373227ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:58.081096911Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_77bd825f-cbb2-4b31-ba9f-d69e9e4cff59","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_a6c98ed9-62e0-43ae-9066-c1d4ee8e0596\") failed: Property not found. retry in 568.181403ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:58.223551777Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_a6c98ed9-62e0-43ae-9066-c1d4ee8e0596","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f\") failed: Property not found. retry in 1.301393727s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:58.324745694Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_82a65d0c-f391-4b1f-9c11-e3a397904f10\") failed: Property not found. retry in 854.319219ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:58.566212187Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_82a65d0c-f391-4b1f-9c11-e3a397904f10","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_77bd825f-cbb2-4b31-ba9f-d69e9e4cff59\") failed: Property not found. retry in 813.479556ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:58.605365377Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_77bd825f-cbb2-4b31-ba9f-d69e9e4cff59","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_a6c98ed9-62e0-43ae-9066-c1d4ee8e0596\") failed: Property not found. retry in 1.470305172s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:58.802188311Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_a6c98ed9-62e0-43ae-9066-c1d4ee8e0596","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_788bf241-055b-45bf-8d67-e865c6324e46\") failed: Property not found. retry in 978.772862ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:58.830068549Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_788bf241-055b-45bf-8d67-e865c6324e46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_77bd825f-cbb2-4b31-ba9f-d69e9e4cff59\") failed: Property not found. retry in 973.650171ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:59.431027363Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_77bd825f-cbb2-4b31-ba9f-d69e9e4cff59","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_82a65d0c-f391-4b1f-9c11-e3a397904f10\") failed: Property not found. retry in 1.148123373s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:59.439389885Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_82a65d0c-f391-4b1f-9c11-e3a397904f10","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f\") failed: Property not found. retry in 586.096344ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:59.639117867Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_788bf241-055b-45bf-8d67-e865c6324e46\") failed: Property not found. retry in 1.278931854s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:23:59.820916834Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_788bf241-055b-45bf-8d67-e865c6324e46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f\") failed: Property not found. retry in 1.217913414s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:00.237649342Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_a6c98ed9-62e0-43ae-9066-c1d4ee8e0596\") failed: Property not found. retry in 1.227134879s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:00.285818806Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_a6c98ed9-62e0-43ae-9066-c1d4ee8e0596","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_77bd825f-cbb2-4b31-ba9f-d69e9e4cff59\") failed: Property not found. retry in 1.487082857s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:00.418035141Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_77bd825f-cbb2-4b31-ba9f-d69e9e4cff59","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_82a65d0c-f391-4b1f-9c11-e3a397904f10\") failed: Property not found. retry in 581.722411ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:00.600594539Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_82a65d0c-f391-4b1f-9c11-e3a397904f10","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_788bf241-055b-45bf-8d67-e865c6324e46\") failed: Property not found. retry in 1.333990302s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:01.115334718Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_788bf241-055b-45bf-8d67-e865c6324e46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_82a65d0c-f391-4b1f-9c11-e3a397904f10\") failed: Property not found. retry in 1.059194884s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:01.192635359Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_82a65d0c-f391-4b1f-9c11-e3a397904f10","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f\") failed: Property not found. retry in 814.004788ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:01.468717574Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_a6c98ed9-62e0-43ae-9066-c1d4ee8e0596\") failed: Property not found. retry in 550.715697ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:01.525925328Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_a6c98ed9-62e0-43ae-9066-c1d4ee8e0596","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_77bd825f-cbb2-4b31-ba9f-d69e9e4cff59\") failed: Property not found. retry in 812.385083ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:01.918724551Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_77bd825f-cbb2-4b31-ba9f-d69e9e4cff59","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_a6c98ed9-62e0-43ae-9066-c1d4ee8e0596\") failed: Property not found. retry in 737.676242ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:02.08694155Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_a6c98ed9-62e0-43ae-9066-c1d4ee8e0596","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_82a65d0c-f391-4b1f-9c11-e3a397904f10\") failed: Property not found. retry in 1.417067547s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:02.264243727Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_82a65d0c-f391-4b1f-9c11-e3a397904f10","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f\") failed: Property not found. retry in 919.142398ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:02.294246712Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_788bf241-055b-45bf-8d67-e865c6324e46\") failed: Property not found. retry in 1.057583935s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:02.462804722Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_788bf241-055b-45bf-8d67-e865c6324e46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_77bd825f-cbb2-4b31-ba9f-d69e9e4cff59\") failed: Property not found. retry in 647.007863ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:02.743668287Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_77bd825f-cbb2-4b31-ba9f-d69e9e4cff59","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_a6c98ed9-62e0-43ae-9066-c1d4ee8e0596\") failed: Property not found. retry in 540.038523ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:02.835815437Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_a6c98ed9-62e0-43ae-9066-c1d4ee8e0596","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:02.847925018Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:60163"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:02.848162981Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"03ddab35-eb80-425b-adb3-65b02a7a30f6","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:60163","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":103,"response_code":"200"} {"msg":"wait for service Some(\"oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f\") failed: Property not found. retry in 1.017736836s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:03.225869949Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_a6c98ed9-62e0-43ae-9066-c1d4ee8e0596\") failed: Property not found. retry in 676.11543ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:03.386214941Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_a6c98ed9-62e0-43ae-9066-c1d4ee8e0596","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_77bd825f-cbb2-4b31-ba9f-d69e9e4cff59\") failed: Property not found. retry in 697.445508ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:03.40119909Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_77bd825f-cbb2-4b31-ba9f-d69e9e4cff59","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_788bf241-055b-45bf-8d67-e865c6324e46\") failed: Property not found. retry in 1.247143605s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:03.533038173Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_788bf241-055b-45bf-8d67-e865c6324e46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_82a65d0c-f391-4b1f-9c11-e3a397904f10\") failed: Property not found. retry in 792.969783ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:03.694392239Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_82a65d0c-f391-4b1f-9c11-e3a397904f10","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_a6c98ed9-62e0-43ae-9066-c1d4ee8e0596\") failed: Property not found. retry in 722.139336ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:04.073801324Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_a6c98ed9-62e0-43ae-9066-c1d4ee8e0596","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f\") failed: Property not found. retry in 636.343412ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:04.256395421Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_cockroachdb_788bf241-055b-45bf-8d67-e865c6324e46\") failed: Property not found. retry in 919.82314ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:04.793023205Z","hostname":"sock","pid":1566,"zone":"oxz_cockroachdb_788bf241-055b-45bf-8d67-e865c6324e46","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:05.738341541Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_cockroachdb_77bd825f-cbb2-4b31-ba9f-d69e9e4cff59"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:05.73839034Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:05.738398252Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_cockroachdb_788bf241-055b-45bf-8d67-e865c6324e46"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:05.738404355Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_cockroachdb_a6c98ed9-62e0-43ae-9066-c1d4ee8e0596"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:05.738411518Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_cockroachdb_82a65d0c-f391-4b1f-9c11-e3a397904f10"} {"msg":"Writing ledger to /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:05.738423828Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:05.73873864Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:05.739053123Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"21d8587e-fd27-40b2-866a-da2fadf1a828","remote_addr":"[fd00:1122:3344:101::1]:46046","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":14210344,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:05.772139448Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:39713"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:05.802428544Z","hostname":"sock","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"5d27f4b4-24de-4b6e-943e-9d28aaf751bc","remote_addr":"[fd00:1122:3344:101::1]:39713","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService6/omicron6 in oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f /usr/sbin/ipadm show-addr -p -o ADDR oxControlService6/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not get address: Object not found","latency_us":29984,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:05.802686608Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1158","retry_after":"119.200854ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5d27f4b4-24de-4b6e-943e-9d28aaf751bc\", \"content-length\": \"124\", \"date\": \"Wed, 15 Jan 2025 02:24:05 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5d27f4b4-24de-4b6e-943e-9d28aaf751bc\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:05.95146053Z","hostname":"sock","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"cf42a18b-3c7e-4849-95e7-8d13fdadbf15","remote_addr":"[fd00:1122:3344:101::1]:39713","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService6/omicron6 in oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f /usr/sbin/ipadm show-addr -p -o ADDR oxControlService6/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not get address: Object not found","latency_us":28199,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:05.951622525Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1158","retry_after":"90.970805ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"cf42a18b-3c7e-4849-95e7-8d13fdadbf15\", \"content-length\": \"124\", \"date\": \"Wed, 15 Jan 2025 02:24:05 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"cf42a18b-3c7e-4849-95e7-8d13fdadbf15\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:06.07080584Z","hostname":"sock","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"5bb16054-e931-47f9-ad3d-507c896c15f6","remote_addr":"[fd00:1122:3344:101::1]:39713","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService6/omicron6 in oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f /usr/sbin/ipadm show-addr -p -o ADDR oxControlService6/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not get address: Object not found","latency_us":27122,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:06.071177336Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1158","retry_after":"123.846805ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5bb16054-e931-47f9-ad3d-507c896c15f6\", \"content-length\": \"124\", \"date\": \"Wed, 15 Jan 2025 02:24:05 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5bb16054-e931-47f9-ad3d-507c896c15f6\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:06.222560612Z","hostname":"sock","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"b4fba9d5-c80a-4363-b5ef-07756f3366fb","remote_addr":"[fd00:1122:3344:101::1]:39713","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService6/omicron6 in oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f /usr/sbin/ipadm show-addr -p -o ADDR oxControlService6/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not get address: Object not found","latency_us":26768,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:06.222738715Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1158","retry_after":"216.057528ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b4fba9d5-c80a-4363-b5ef-07756f3366fb\", \"content-length\": \"124\", \"date\": \"Wed, 15 Jan 2025 02:24:05 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b4fba9d5-c80a-4363-b5ef-07756f3366fb\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:06.467038281Z","hostname":"sock","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"1602840b-009b-4d67-8b25-fc12222f2b3f","remote_addr":"[fd00:1122:3344:101::1]:39713","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService6/omicron6 in oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f /usr/sbin/ipadm show-addr -p -o ADDR oxControlService6/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not get address: Object not found","latency_us":26477,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:06.467250665Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1158","retry_after":"299.168867ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1602840b-009b-4d67-8b25-fc12222f2b3f\", \"content-length\": \"124\", \"date\": \"Wed, 15 Jan 2025 02:24:05 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1602840b-009b-4d67-8b25-fc12222f2b3f\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:06.794658967Z","hostname":"sock","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"77b22641-40c3-45ce-b2dc-2e3f504de2d9","remote_addr":"[fd00:1122:3344:101::1]:39713","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService6/omicron6 in oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f /usr/sbin/ipadm show-addr -p -o ADDR oxControlService6/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not get address: Object not found","latency_us":25843,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:06.79487096Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1158","retry_after":"319.085853ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"77b22641-40c3-45ce-b2dc-2e3f504de2d9\", \"content-length\": \"124\", \"date\": \"Wed, 15 Jan 2025 02:24:06 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"77b22641-40c3-45ce-b2dc-2e3f504de2d9\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:07.142504153Z","hostname":"sock","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"2670a45e-5f50-4658-823c-30bb017e4adb","remote_addr":"[fd00:1122:3344:101::1]:39713","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService6/omicron6 in oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f /usr/sbin/ipadm show-addr -p -o ADDR oxControlService6/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not get address: Object not found","latency_us":26302,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:07.142680342Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1158","retry_after":"153.407462ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2670a45e-5f50-4658-823c-30bb017e4adb\", \"content-length\": \"124\", \"date\": \"Wed, 15 Jan 2025 02:24:06 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2670a45e-5f50-4658-823c-30bb017e4adb\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:07.32636313Z","hostname":"sock","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"d9025754-8986-4b73-878f-0e120a965113","remote_addr":"[fd00:1122:3344:101::1]:39713","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService6/omicron6 in oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f /usr/sbin/ipadm show-addr -p -o ADDR oxControlService6/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not get address: Object not found","latency_us":27931,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:07.326721259Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1158","retry_after":"180.317491ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d9025754-8986-4b73-878f-0e120a965113\", \"content-length\": \"124\", \"date\": \"Wed, 15 Jan 2025 02:24:06 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d9025754-8986-4b73-878f-0e120a965113\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:07.534459171Z","hostname":"sock","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"f62fa001-7122-4f62-a921-1b8c24308458","remote_addr":"[fd00:1122:3344:101::1]:39713","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService6/omicron6 in oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f /usr/sbin/ipadm show-addr -p -o ADDR oxControlService6/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not get address: Object not found","latency_us":25939,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:07.534626772Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1158","retry_after":"644.368087ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f62fa001-7122-4f62-a921-1b8c24308458\", \"content-length\": \"124\", \"date\": \"Wed, 15 Jan 2025 02:24:07 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f62fa001-7122-4f62-a921-1b8c24308458\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:08.208702018Z","hostname":"sock","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"b8e28684-3d69-4fa7-b0c3-8ba775450933","remote_addr":"[fd00:1122:3344:101::1]:39713","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService6/omicron6 in oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f /usr/sbin/ipadm show-addr -p -o ADDR oxControlService6/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not get address: Object not found","latency_us":26855,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:08.208928603Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1158","retry_after":"458.562391ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b8e28684-3d69-4fa7-b0c3-8ba775450933\", \"content-length\": \"124\", \"date\": \"Wed, 15 Jan 2025 02:24:07 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b8e28684-3d69-4fa7-b0c3-8ba775450933\" }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:08.705677187Z","hostname":"sock","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"df642f7e-8a55-45c5-b486-5ec8cde9f4be","remote_addr":"[fd00:1122:3344:101::1]:39713","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService6/omicron6 in oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_03e9536c-5029-447f-944a-4daff2909c7f /usr/sbin/ipadm show-addr -p -o ADDR oxControlService6/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not get address: Object not found","latency_us":36109,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:08.705829105Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1158","retry_after":"311.536558ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"df642f7e-8a55-45c5-b486-5ec8cde9f4be\", \"content-length\": \"124\", \"date\": \"Wed, 15 Jan 2025 02:24:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"df642f7e-8a55-45c5-b486-5ec8cde9f4be\" }"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::7]:32221","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:09.055162932Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3758"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:27.337935961Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3776"} {"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.254799379Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3799"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.256566847Z","hostname":"sock","pid":1566,"uri":"/cockroachdb","method":"POST","req_id":"0ed4420f-8b69-48b7-8e70-f3e35d236edb","remote_addr":"[fd00:1122:3344:101::1]:39713","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":27163135,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.259053307Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:58993"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.260345363Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"05ba246d-b659-425f-84f9-6fec0bb388bd","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:58993","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":478,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:36.26856311Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"631.562873ms"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e94e881-2bab-4d68-8315-741d3cd1814a (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a4d0b75d-0c67-4224-af19-0510665c8775 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 44ac331a-da63-47e3-a636-214c3ca86b22 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 46c4b5d7-b839-4d27-ac2d-e1f6f6eae7d3, kind: Service { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 }, name: Name(\"ntp-fd5ec85c-1d4a-43a3-bd64-3bf287ecded3\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 9])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 77bd825f-cbb2-4b31-ba9f-d69e9e4cff59 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: a6c98ed9-62e0-43ae-9066-c1d4ee8e0596 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 788bf241-055b-45bf-8d67-e865c6324e46 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 82a65d0c-f391-4b1f-9c11-e3a397904f10 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 03e9536c-5029-447f-944a-4daff2909c7f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7468547b-b2f3-4993-afd8-72e9164a4301 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: b15378a9-5120-477b-abef-a170753103c1, kind: Service { id: 7468547b-b2f3-4993-afd8-72e9164a4301 }, name: Name(\"external-dns-7468547b-b2f3-4993-afd8-72e9164a4301\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 124])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beea2870-fa6b-4d58-b0eb-af5a33149d11, kind: Service { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a }, name: Name(\"external-dns-8b302e6b-b09c-4ef2-8768-1e5fae50d31a\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 41])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 357305ec-a785-4a6e-96cf-879dc3168076, kind: Service { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f }, name: Name(\"nexus-a1e38c68-2155-4ce9-b70a-66d434b3f24f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 213])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 56f43fd0-1c39-449d-9089-5f10b7acf59b, kind: Service { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d }, name: Name(\"nexus-b4bce764-527e-4a0d-9efa-46f5760fbb1d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 31])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 743f2600-b4a7-4963-8bef-692428b6800a (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 22195cd0-81ca-4c14-8294-e0bb8767fa51, kind: Service { id: 743f2600-b4a7-4963-8bef-692428b6800a }, name: Name(\"nexus-743f2600-b4a7-4963-8bef-692428b6800a\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 167, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f71d33aa-7320-48b9-81d2-899a100e78cd (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 40c9a9db-bd0a-45df-b2ea-f8a9ad278c76 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: eb0487cd-afd0-473b-b56c-c84ebdb9a402 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 683f3bcd-05eb-48ca-a731-4591f3875c45 (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 100f148b-d0a5-4957-b7f4-7b142438883b (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 1ecfbd35-39cb-454a-b9bf-b046feb4f811 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: 014247db-8886-4cf5-b68e-375be0dabc7f (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 7361bad6-7bc4-4252-9897-84b3278d100e (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6e9d55e6-6648-4adf-a2be-39839524cd72 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ceaca86-5562-442f-a7a7-bbfd684c302b (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 771664a2-5c9b-4063-8f72-848056c94344 (service), filesystem_pool: Some(ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37091b30-13af-44d8-ac11-483eada5eeca (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 25c723ea-c52b-44b6-8751-dbe925d1e7a7 (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External } } } }, OmicronZoneConfig { id: f12a4089-4d9a-4d0c-b3e6-8a12d533bb03 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.304681233Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.305695267Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:53729"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.306992575Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.307004303Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.32022126Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.320266383Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.32914741Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.329189861Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.339590567Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.339692406Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.35006455Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.350105651Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.360824981Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.3608652Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.370144944Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.370185642Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.380161648Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.380198502Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.389963387Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.390007638Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.399869181Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.399916985Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.41113277Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.411169043Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.423666702Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.423707203Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.43342095Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.433564242Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.443520592Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.443555899Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.45413942Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.454182443Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.464506435Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.464544827Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.473881663Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.473921349Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.483555382Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.483590624Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.493783152Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.493824075Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.50452752Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.504564374Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.51418822Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.514237057Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.514645489Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.51494125Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:36.63718354Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:36.637228533Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:36.637232723Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:36.63723683Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:36.637240983Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:36.63724418Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:36.637248136Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:36.637253076Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:36.63725617Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:36.637260089Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:36.637263268Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:36.637266277Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:36.63727014Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:36.637274049Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:36.637278081Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:36.637296522Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:36.637300685Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:36.637304781Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:36.6373087Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.637381143Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"a85c9117-3728-4d9d-91de-1b9baa4ea65b","remote_addr":"[fd00:1122:3344:101::1]:53729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd\", TimeNotSynchronized), (\"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344\", TimeNotSynchronized), (\"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f\", TimeNotSynchronized), (\"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402\", TimeNotSynchronized), (\"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a\", TimeNotSynchronized), (\"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\", TimeNotSynchronized), (\"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e\", TimeNotSynchronized), (\"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811\", TimeNotSynchronized), (\"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b\", TimeNotSynchronized), (\"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301\", TimeNotSynchronized), (\"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca\", TimeNotSynchronized), (\"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45\", TimeNotSynchronized), (\"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\", TimeNotSynchronized), (\"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a\", TimeNotSynchronized), (\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\", TimeNotSynchronized), (\"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7\", TimeNotSynchronized), (\"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b\", TimeNotSynchronized), (\"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d\", TimeNotSynchronized), (\"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\", TimeNotSynchronized)]","latency_us":330572,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:36.637619576Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"77.815791ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a85c9117-3728-4d9d-91de-1b9baa4ea65b\", \"content-length\": \"124\", \"date\": \"Wed, 15 Jan 2025 02:24:36 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a85c9117-3728-4d9d-91de-1b9baa4ea65b\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e94e881-2bab-4d68-8315-741d3cd1814a (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a4d0b75d-0c67-4224-af19-0510665c8775 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 44ac331a-da63-47e3-a636-214c3ca86b22 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 46c4b5d7-b839-4d27-ac2d-e1f6f6eae7d3, kind: Service { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 }, name: Name(\"ntp-fd5ec85c-1d4a-43a3-bd64-3bf287ecded3\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 9])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 77bd825f-cbb2-4b31-ba9f-d69e9e4cff59 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: a6c98ed9-62e0-43ae-9066-c1d4ee8e0596 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 788bf241-055b-45bf-8d67-e865c6324e46 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 82a65d0c-f391-4b1f-9c11-e3a397904f10 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 03e9536c-5029-447f-944a-4daff2909c7f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7468547b-b2f3-4993-afd8-72e9164a4301 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: b15378a9-5120-477b-abef-a170753103c1, kind: Service { id: 7468547b-b2f3-4993-afd8-72e9164a4301 }, name: Name(\"external-dns-7468547b-b2f3-4993-afd8-72e9164a4301\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 124])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beea2870-fa6b-4d58-b0eb-af5a33149d11, kind: Service { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a }, name: Name(\"external-dns-8b302e6b-b09c-4ef2-8768-1e5fae50d31a\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 41])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 357305ec-a785-4a6e-96cf-879dc3168076, kind: Service { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f }, name: Name(\"nexus-a1e38c68-2155-4ce9-b70a-66d434b3f24f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 213])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 56f43fd0-1c39-449d-9089-5f10b7acf59b, kind: Service { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d }, name: Name(\"nexus-b4bce764-527e-4a0d-9efa-46f5760fbb1d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 31])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 743f2600-b4a7-4963-8bef-692428b6800a (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 22195cd0-81ca-4c14-8294-e0bb8767fa51, kind: Service { id: 743f2600-b4a7-4963-8bef-692428b6800a }, name: Name(\"nexus-743f2600-b4a7-4963-8bef-692428b6800a\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 167, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f71d33aa-7320-48b9-81d2-899a100e78cd (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 40c9a9db-bd0a-45df-b2ea-f8a9ad278c76 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: eb0487cd-afd0-473b-b56c-c84ebdb9a402 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 683f3bcd-05eb-48ca-a731-4591f3875c45 (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 100f148b-d0a5-4957-b7f4-7b142438883b (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 1ecfbd35-39cb-454a-b9bf-b046feb4f811 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: 014247db-8886-4cf5-b68e-375be0dabc7f (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 7361bad6-7bc4-4252-9897-84b3278d100e (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6e9d55e6-6648-4adf-a2be-39839524cd72 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ceaca86-5562-442f-a7a7-bbfd684c302b (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 771664a2-5c9b-4063-8f72-848056c94344 (service), filesystem_pool: Some(ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37091b30-13af-44d8-ac11-483eada5eeca (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 25c723ea-c52b-44b6-8751-dbe925d1e7a7 (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External } } } }, OmicronZoneConfig { id: f12a4089-4d9a-4d0c-b3e6-8a12d533bb03 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.716095046Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.71651442Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.716522117Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.727003174Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.727039278Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.735177178Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.735209889Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.745553461Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.74558768Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.754225582Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.754258911Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.764119246Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.764155097Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.772764505Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.772876595Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.781239744Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.781273524Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.790914715Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.790948794Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.800741801Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.800780727Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.809811441Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.809844722Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.818152378Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.818190338Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.828119095Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.828150803Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.837136931Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.837175688Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.84548821Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.845525823Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.855285324Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.855323191Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.864097824Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.864140838Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.874431927Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.874466793Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.883573414Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.883609771Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.891880466Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.891914948Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.900444831Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.900496076Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.900717116Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:36.900870327Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.003791899Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.003846529Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.003852791Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.003857732Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.003863095Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.003868504Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.003874888Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.003878817Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.003885005Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.003890396Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.003895477Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.003898833Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.003910018Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.003915456Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.003918606Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.003923312Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.003926584Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.003931196Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.003934571Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.003974501Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"1b43568a-abeb-486b-93dd-5b1b6c65dc84","remote_addr":"[fd00:1122:3344:101::1]:53729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b\", TimeNotSynchronized), (\"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811\", TimeNotSynchronized), (\"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b\", TimeNotSynchronized), (\"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344\", TimeNotSynchronized), (\"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca\", TimeNotSynchronized), (\"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7\", TimeNotSynchronized), (\"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402\", TimeNotSynchronized), (\"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45\", TimeNotSynchronized), (\"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301\", TimeNotSynchronized), (\"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e\", TimeNotSynchronized), (\"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\", TimeNotSynchronized), (\"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a\", TimeNotSynchronized), (\"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a\", TimeNotSynchronized), (\"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\", TimeNotSynchronized), (\"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f\", TimeNotSynchronized), (\"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\", TimeNotSynchronized), (\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\", TimeNotSynchronized), (\"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd\", TimeNotSynchronized), (\"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d\", TimeNotSynchronized)]","latency_us":286858,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.00412758Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"156.927658ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1b43568a-abeb-486b-93dd-5b1b6c65dc84\", \"content-length\": \"124\", \"date\": \"Wed, 15 Jan 2025 02:24:37 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1b43568a-abeb-486b-93dd-5b1b6c65dc84\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e94e881-2bab-4d68-8315-741d3cd1814a (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a4d0b75d-0c67-4224-af19-0510665c8775 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 44ac331a-da63-47e3-a636-214c3ca86b22 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 46c4b5d7-b839-4d27-ac2d-e1f6f6eae7d3, kind: Service { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 }, name: Name(\"ntp-fd5ec85c-1d4a-43a3-bd64-3bf287ecded3\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 9])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 77bd825f-cbb2-4b31-ba9f-d69e9e4cff59 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: a6c98ed9-62e0-43ae-9066-c1d4ee8e0596 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 788bf241-055b-45bf-8d67-e865c6324e46 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 82a65d0c-f391-4b1f-9c11-e3a397904f10 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 03e9536c-5029-447f-944a-4daff2909c7f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7468547b-b2f3-4993-afd8-72e9164a4301 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: b15378a9-5120-477b-abef-a170753103c1, kind: Service { id: 7468547b-b2f3-4993-afd8-72e9164a4301 }, name: Name(\"external-dns-7468547b-b2f3-4993-afd8-72e9164a4301\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 124])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beea2870-fa6b-4d58-b0eb-af5a33149d11, kind: Service { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a }, name: Name(\"external-dns-8b302e6b-b09c-4ef2-8768-1e5fae50d31a\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 41])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 357305ec-a785-4a6e-96cf-879dc3168076, kind: Service { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f }, name: Name(\"nexus-a1e38c68-2155-4ce9-b70a-66d434b3f24f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 213])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 56f43fd0-1c39-449d-9089-5f10b7acf59b, kind: Service { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d }, name: Name(\"nexus-b4bce764-527e-4a0d-9efa-46f5760fbb1d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 31])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 743f2600-b4a7-4963-8bef-692428b6800a (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 22195cd0-81ca-4c14-8294-e0bb8767fa51, kind: Service { id: 743f2600-b4a7-4963-8bef-692428b6800a }, name: Name(\"nexus-743f2600-b4a7-4963-8bef-692428b6800a\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 167, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f71d33aa-7320-48b9-81d2-899a100e78cd (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 40c9a9db-bd0a-45df-b2ea-f8a9ad278c76 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: eb0487cd-afd0-473b-b56c-c84ebdb9a402 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 683f3bcd-05eb-48ca-a731-4591f3875c45 (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 100f148b-d0a5-4957-b7f4-7b142438883b (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 1ecfbd35-39cb-454a-b9bf-b046feb4f811 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: 014247db-8886-4cf5-b68e-375be0dabc7f (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 7361bad6-7bc4-4252-9897-84b3278d100e (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6e9d55e6-6648-4adf-a2be-39839524cd72 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ceaca86-5562-442f-a7a7-bbfd684c302b (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 771664a2-5c9b-4063-8f72-848056c94344 (service), filesystem_pool: Some(ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37091b30-13af-44d8-ac11-483eada5eeca (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 25c723ea-c52b-44b6-8751-dbe925d1e7a7 (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External } } } }, OmicronZoneConfig { id: f12a4089-4d9a-4d0c-b3e6-8a12d533bb03 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.163172488Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.163671579Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.163678996Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.175394077Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.175434991Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.186162573Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.186199043Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.195977108Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.196011853Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.20562284Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.205660463Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.215862012Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.215895125Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.225199155Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.225231331Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.234514548Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.234548853Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.243601852Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.243635116Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.253232336Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.25326575Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.263552344Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.263591514Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.273281982Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.273316558Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.281792509Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.281825923Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.290354001Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.290387725Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.298494081Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.2985283Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.306668931Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.30670046Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.315162655Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.315195796Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.323570672Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.323606064Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.331782627Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.331816228Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.341636995Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.341669462Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.350264416Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.350306239Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.35057693Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.350692753Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.443817393Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.443874066Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.443884595Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.443889883Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.443894908Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.443901826Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.443908951Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.443914586Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.443918196Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.443937096Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.443944155Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.44394933Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.443955959Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.443961284Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.443966328Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.443973613Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.443978684Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.443983597Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.443987067Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.444191268Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"97451e3c-c296-4aae-a0c6-ebe2743b3cd7","remote_addr":"[fd00:1122:3344:101::1]:53729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b\", TimeNotSynchronized), (\"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344\", TimeNotSynchronized), (\"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca\", TimeNotSynchronized), (\"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a\", TimeNotSynchronized), (\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\", TimeNotSynchronized), (\"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b\", TimeNotSynchronized), (\"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\", TimeNotSynchronized), (\"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\", TimeNotSynchronized), (\"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f\", TimeNotSynchronized), (\"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402\", TimeNotSynchronized), (\"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7\", TimeNotSynchronized), (\"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a\", TimeNotSynchronized), (\"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d\", TimeNotSynchronized), (\"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd\", TimeNotSynchronized), (\"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811\", TimeNotSynchronized), (\"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\", TimeNotSynchronized), (\"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e\", TimeNotSynchronized), (\"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301\", TimeNotSynchronized), (\"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45\", TimeNotSynchronized)]","latency_us":279784,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.444365479Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"148.025406ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"97451e3c-c296-4aae-a0c6-ebe2743b3cd7\", \"content-length\": \"124\", \"date\": \"Wed, 15 Jan 2025 02:24:37 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"97451e3c-c296-4aae-a0c6-ebe2743b3cd7\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e94e881-2bab-4d68-8315-741d3cd1814a (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a4d0b75d-0c67-4224-af19-0510665c8775 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 44ac331a-da63-47e3-a636-214c3ca86b22 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 46c4b5d7-b839-4d27-ac2d-e1f6f6eae7d3, kind: Service { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 }, name: Name(\"ntp-fd5ec85c-1d4a-43a3-bd64-3bf287ecded3\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 9])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 77bd825f-cbb2-4b31-ba9f-d69e9e4cff59 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: a6c98ed9-62e0-43ae-9066-c1d4ee8e0596 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 788bf241-055b-45bf-8d67-e865c6324e46 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 82a65d0c-f391-4b1f-9c11-e3a397904f10 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 03e9536c-5029-447f-944a-4daff2909c7f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7468547b-b2f3-4993-afd8-72e9164a4301 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: b15378a9-5120-477b-abef-a170753103c1, kind: Service { id: 7468547b-b2f3-4993-afd8-72e9164a4301 }, name: Name(\"external-dns-7468547b-b2f3-4993-afd8-72e9164a4301\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 124])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beea2870-fa6b-4d58-b0eb-af5a33149d11, kind: Service { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a }, name: Name(\"external-dns-8b302e6b-b09c-4ef2-8768-1e5fae50d31a\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 41])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 357305ec-a785-4a6e-96cf-879dc3168076, kind: Service { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f }, name: Name(\"nexus-a1e38c68-2155-4ce9-b70a-66d434b3f24f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 213])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 56f43fd0-1c39-449d-9089-5f10b7acf59b, kind: Service { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d }, name: Name(\"nexus-b4bce764-527e-4a0d-9efa-46f5760fbb1d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 31])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 743f2600-b4a7-4963-8bef-692428b6800a (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 22195cd0-81ca-4c14-8294-e0bb8767fa51, kind: Service { id: 743f2600-b4a7-4963-8bef-692428b6800a }, name: Name(\"nexus-743f2600-b4a7-4963-8bef-692428b6800a\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 167, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f71d33aa-7320-48b9-81d2-899a100e78cd (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 40c9a9db-bd0a-45df-b2ea-f8a9ad278c76 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: eb0487cd-afd0-473b-b56c-c84ebdb9a402 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 683f3bcd-05eb-48ca-a731-4591f3875c45 (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 100f148b-d0a5-4957-b7f4-7b142438883b (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 1ecfbd35-39cb-454a-b9bf-b046feb4f811 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: 014247db-8886-4cf5-b68e-375be0dabc7f (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 7361bad6-7bc4-4252-9897-84b3278d100e (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6e9d55e6-6648-4adf-a2be-39839524cd72 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ceaca86-5562-442f-a7a7-bbfd684c302b (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 771664a2-5c9b-4063-8f72-848056c94344 (service), filesystem_pool: Some(ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37091b30-13af-44d8-ac11-483eada5eeca (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 25c723ea-c52b-44b6-8751-dbe925d1e7a7 (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External } } } }, OmicronZoneConfig { id: f12a4089-4d9a-4d0c-b3e6-8a12d533bb03 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.59405801Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.594470942Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.59447925Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.606468945Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.60651585Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.616743667Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.616784449Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.627956966Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.628003402Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.637695925Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.63773757Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.647027572Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.647068945Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.656222886Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.656268787Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.665356117Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.665392483Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.675780718Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.675816503Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.68544825Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.685489894Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.694889198Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.694928189Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.704193197Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.704232236Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.713517824Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.713555334Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.723859786Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.723897953Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.73410579Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.734147491Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.745345947Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.745386701Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.755658844Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.755705485Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.766519327Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.766562538Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.776372607Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.776429645Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.786092518Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.786131904Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.795326717Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.795375121Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.795655499Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.795785271Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.893933837Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.893984633Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.893991927Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.89399742Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.894003899Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.894008802Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.894019434Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.89402324Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.894027881Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.894032999Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.894037996Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.894041371Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.894048197Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.894053025Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.894056578Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.894060188Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.894064678Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.894068166Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.894073051Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:37.894104326Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"c1bdca81-9e83-4273-ba02-e7ed6eb66d6b","remote_addr":"[fd00:1122:3344:101::1]:53729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301\", TimeNotSynchronized), (\"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\", TimeNotSynchronized), (\"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45\", TimeNotSynchronized), (\"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344\", TimeNotSynchronized), (\"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7\", TimeNotSynchronized), (\"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a\", TimeNotSynchronized), (\"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca\", TimeNotSynchronized), (\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\", TimeNotSynchronized), (\"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e\", TimeNotSynchronized), (\"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d\", TimeNotSynchronized), (\"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f\", TimeNotSynchronized), (\"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b\", TimeNotSynchronized), (\"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811\", TimeNotSynchronized), (\"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\", TimeNotSynchronized), (\"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402\", TimeNotSynchronized), (\"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd\", TimeNotSynchronized), (\"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a\", TimeNotSynchronized), (\"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\", TimeNotSynchronized), (\"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b\", TimeNotSynchronized)]","latency_us":298983,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:37.894255568Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"120.019141ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c1bdca81-9e83-4273-ba02-e7ed6eb66d6b\", \"content-length\": \"124\", \"date\": \"Wed, 15 Jan 2025 02:24:37 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c1bdca81-9e83-4273-ba02-e7ed6eb66d6b\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e94e881-2bab-4d68-8315-741d3cd1814a (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a4d0b75d-0c67-4224-af19-0510665c8775 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 44ac331a-da63-47e3-a636-214c3ca86b22 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 46c4b5d7-b839-4d27-ac2d-e1f6f6eae7d3, kind: Service { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 }, name: Name(\"ntp-fd5ec85c-1d4a-43a3-bd64-3bf287ecded3\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 9])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 77bd825f-cbb2-4b31-ba9f-d69e9e4cff59 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: a6c98ed9-62e0-43ae-9066-c1d4ee8e0596 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 788bf241-055b-45bf-8d67-e865c6324e46 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 82a65d0c-f391-4b1f-9c11-e3a397904f10 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 03e9536c-5029-447f-944a-4daff2909c7f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7468547b-b2f3-4993-afd8-72e9164a4301 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: b15378a9-5120-477b-abef-a170753103c1, kind: Service { id: 7468547b-b2f3-4993-afd8-72e9164a4301 }, name: Name(\"external-dns-7468547b-b2f3-4993-afd8-72e9164a4301\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 124])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beea2870-fa6b-4d58-b0eb-af5a33149d11, kind: Service { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a }, name: Name(\"external-dns-8b302e6b-b09c-4ef2-8768-1e5fae50d31a\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 41])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 357305ec-a785-4a6e-96cf-879dc3168076, kind: Service { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f }, name: Name(\"nexus-a1e38c68-2155-4ce9-b70a-66d434b3f24f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 213])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 56f43fd0-1c39-449d-9089-5f10b7acf59b, kind: Service { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d }, name: Name(\"nexus-b4bce764-527e-4a0d-9efa-46f5760fbb1d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 31])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 743f2600-b4a7-4963-8bef-692428b6800a (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 22195cd0-81ca-4c14-8294-e0bb8767fa51, kind: Service { id: 743f2600-b4a7-4963-8bef-692428b6800a }, name: Name(\"nexus-743f2600-b4a7-4963-8bef-692428b6800a\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 167, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f71d33aa-7320-48b9-81d2-899a100e78cd (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 40c9a9db-bd0a-45df-b2ea-f8a9ad278c76 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: eb0487cd-afd0-473b-b56c-c84ebdb9a402 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 683f3bcd-05eb-48ca-a731-4591f3875c45 (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 100f148b-d0a5-4957-b7f4-7b142438883b (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 1ecfbd35-39cb-454a-b9bf-b046feb4f811 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: 014247db-8886-4cf5-b68e-375be0dabc7f (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 7361bad6-7bc4-4252-9897-84b3278d100e (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6e9d55e6-6648-4adf-a2be-39839524cd72 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ceaca86-5562-442f-a7a7-bbfd684c302b (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 771664a2-5c9b-4063-8f72-848056c94344 (service), filesystem_pool: Some(ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37091b30-13af-44d8-ac11-483eada5eeca (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 25c723ea-c52b-44b6-8751-dbe925d1e7a7 (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External } } } }, OmicronZoneConfig { id: f12a4089-4d9a-4d0c-b3e6-8a12d533bb03 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.016008314Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.016323205Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.016332121Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.026996172Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.02705517Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.038378955Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.038418228Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.047776546Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.047809528Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.057880401Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.057919589Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.067170869Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.067207901Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.075999867Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.076035099Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.08544673Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.0854799Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.095130465Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.095165613Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.10391025Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.103944217Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.113540906Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.113575744Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.122821743Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.12285691Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.130991225Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.131027104Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.139414722Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.139514721Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.150149801Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.150183665Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.160386933Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.160427229Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.16908422Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.169119302Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.179442988Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.179482909Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.189812176Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.189847577Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.200591654Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.200690783Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.2109187Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.210962239Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.211175602Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.21129102Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:38.307110238Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:38.307167475Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:38.30717564Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:38.307197437Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:38.307204515Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:38.30721121Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:38.307216657Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:38.307221682Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:38.307227026Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:38.307230682Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:38.307235783Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:38.30724078Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:38.30724572Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:38.307250942Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:38.307255976Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:38.307260692Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:38.307265595Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:38.307272009Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:38.307276724Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.307363314Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"74769c58-e52a-48bd-8288-78286e8955a5","remote_addr":"[fd00:1122:3344:101::1]:53729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301\", TimeNotSynchronized), (\"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d\", TimeNotSynchronized), (\"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd\", TimeNotSynchronized), (\"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45\", TimeNotSynchronized), (\"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca\", TimeNotSynchronized), (\"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b\", TimeNotSynchronized), (\"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f\", TimeNotSynchronized), (\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\", TimeNotSynchronized), (\"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811\", TimeNotSynchronized), (\"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402\", TimeNotSynchronized), (\"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a\", TimeNotSynchronized), (\"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e\", TimeNotSynchronized), (\"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\", TimeNotSynchronized), (\"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b\", TimeNotSynchronized), (\"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a\", TimeNotSynchronized), (\"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344\", TimeNotSynchronized), (\"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\", TimeNotSynchronized), (\"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7\", TimeNotSynchronized), (\"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\", TimeNotSynchronized)]","latency_us":290446,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:38.307526903Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"207.620961ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"74769c58-e52a-48bd-8288-78286e8955a5\", \"content-length\": \"124\", \"date\": \"Wed, 15 Jan 2025 02:24:37 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"74769c58-e52a-48bd-8288-78286e8955a5\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e94e881-2bab-4d68-8315-741d3cd1814a (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a4d0b75d-0c67-4224-af19-0510665c8775 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 44ac331a-da63-47e3-a636-214c3ca86b22 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 46c4b5d7-b839-4d27-ac2d-e1f6f6eae7d3, kind: Service { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 }, name: Name(\"ntp-fd5ec85c-1d4a-43a3-bd64-3bf287ecded3\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 9])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 77bd825f-cbb2-4b31-ba9f-d69e9e4cff59 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: a6c98ed9-62e0-43ae-9066-c1d4ee8e0596 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 788bf241-055b-45bf-8d67-e865c6324e46 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 82a65d0c-f391-4b1f-9c11-e3a397904f10 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 03e9536c-5029-447f-944a-4daff2909c7f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7468547b-b2f3-4993-afd8-72e9164a4301 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: b15378a9-5120-477b-abef-a170753103c1, kind: Service { id: 7468547b-b2f3-4993-afd8-72e9164a4301 }, name: Name(\"external-dns-7468547b-b2f3-4993-afd8-72e9164a4301\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 124])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beea2870-fa6b-4d58-b0eb-af5a33149d11, kind: Service { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a }, name: Name(\"external-dns-8b302e6b-b09c-4ef2-8768-1e5fae50d31a\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 41])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 357305ec-a785-4a6e-96cf-879dc3168076, kind: Service { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f }, name: Name(\"nexus-a1e38c68-2155-4ce9-b70a-66d434b3f24f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 213])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 56f43fd0-1c39-449d-9089-5f10b7acf59b, kind: Service { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d }, name: Name(\"nexus-b4bce764-527e-4a0d-9efa-46f5760fbb1d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 31])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 743f2600-b4a7-4963-8bef-692428b6800a (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 22195cd0-81ca-4c14-8294-e0bb8767fa51, kind: Service { id: 743f2600-b4a7-4963-8bef-692428b6800a }, name: Name(\"nexus-743f2600-b4a7-4963-8bef-692428b6800a\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 167, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f71d33aa-7320-48b9-81d2-899a100e78cd (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 40c9a9db-bd0a-45df-b2ea-f8a9ad278c76 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: eb0487cd-afd0-473b-b56c-c84ebdb9a402 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 683f3bcd-05eb-48ca-a731-4591f3875c45 (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 100f148b-d0a5-4957-b7f4-7b142438883b (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 1ecfbd35-39cb-454a-b9bf-b046feb4f811 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: 014247db-8886-4cf5-b68e-375be0dabc7f (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 7361bad6-7bc4-4252-9897-84b3278d100e (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6e9d55e6-6648-4adf-a2be-39839524cd72 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ceaca86-5562-442f-a7a7-bbfd684c302b (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 771664a2-5c9b-4063-8f72-848056c94344 (service), filesystem_pool: Some(ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37091b30-13af-44d8-ac11-483eada5eeca (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 25c723ea-c52b-44b6-8751-dbe925d1e7a7 (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External } } } }, OmicronZoneConfig { id: f12a4089-4d9a-4d0c-b3e6-8a12d533bb03 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.517603631Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.517877511Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.517888508Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.529442671Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.529497694Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.539277769Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.539321045Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.548504822Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.548620068Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.55801396Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.558060892Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.568702204Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.568742153Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.577574468Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.577625405Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.586974363Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.587013467Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.59596729Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.596009591Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.605190645Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.605238244Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.614289513Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.614328158Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.624645248Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.624693596Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.635870337Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.635913399Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.64524172Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.645287781Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.654451925Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.654493833Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.664241795Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.664279614Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.673876352Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.673994303Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.683781372Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.68382057Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.694314752Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.694351484Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.704185533Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.70422715Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.713291392Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.713344166Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.713582251Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.713731045Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:38.813192534Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:38.813233476Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:38.813239363Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:38.813243348Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:38.813247305Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:38.813252301Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:38.813257233Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:38.813260533Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:38.813330362Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:38.813334309Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:38.813338397Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:38.81334239Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:38.8133472Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:38.813351232Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:38.813355216Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:38.813359088Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:38.813375382Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:38.813380388Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:38.813384158Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:38.813491963Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"12acd316-e8d5-4300-b635-eb8c0e3ec70f","remote_addr":"[fd00:1122:3344:101::1]:53729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301\", TimeNotSynchronized), (\"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a\", TimeNotSynchronized), (\"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f\", TimeNotSynchronized), (\"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd\", TimeNotSynchronized), (\"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b\", TimeNotSynchronized), (\"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45\", TimeNotSynchronized), (\"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e\", TimeNotSynchronized), (\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\", TimeNotSynchronized), (\"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402\", TimeNotSynchronized), (\"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\", TimeNotSynchronized), (\"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\", TimeNotSynchronized), (\"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b\", TimeNotSynchronized), (\"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d\", TimeNotSynchronized), (\"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811\", TimeNotSynchronized), (\"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344\", TimeNotSynchronized), (\"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca\", TimeNotSynchronized), (\"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a\", TimeNotSynchronized), (\"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7\", TimeNotSynchronized), (\"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\", TimeNotSynchronized)]","latency_us":294986,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:38.813630839Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"256.937614ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"12acd316-e8d5-4300-b635-eb8c0e3ec70f\", \"content-length\": \"124\", \"date\": \"Wed, 15 Jan 2025 02:24:38 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"12acd316-e8d5-4300-b635-eb8c0e3ec70f\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e94e881-2bab-4d68-8315-741d3cd1814a (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a4d0b75d-0c67-4224-af19-0510665c8775 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 44ac331a-da63-47e3-a636-214c3ca86b22 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 46c4b5d7-b839-4d27-ac2d-e1f6f6eae7d3, kind: Service { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 }, name: Name(\"ntp-fd5ec85c-1d4a-43a3-bd64-3bf287ecded3\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 9])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 77bd825f-cbb2-4b31-ba9f-d69e9e4cff59 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: a6c98ed9-62e0-43ae-9066-c1d4ee8e0596 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 788bf241-055b-45bf-8d67-e865c6324e46 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 82a65d0c-f391-4b1f-9c11-e3a397904f10 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 03e9536c-5029-447f-944a-4daff2909c7f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7468547b-b2f3-4993-afd8-72e9164a4301 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: b15378a9-5120-477b-abef-a170753103c1, kind: Service { id: 7468547b-b2f3-4993-afd8-72e9164a4301 }, name: Name(\"external-dns-7468547b-b2f3-4993-afd8-72e9164a4301\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 124])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beea2870-fa6b-4d58-b0eb-af5a33149d11, kind: Service { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a }, name: Name(\"external-dns-8b302e6b-b09c-4ef2-8768-1e5fae50d31a\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 41])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 357305ec-a785-4a6e-96cf-879dc3168076, kind: Service { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f }, name: Name(\"nexus-a1e38c68-2155-4ce9-b70a-66d434b3f24f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 213])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 56f43fd0-1c39-449d-9089-5f10b7acf59b, kind: Service { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d }, name: Name(\"nexus-b4bce764-527e-4a0d-9efa-46f5760fbb1d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 31])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 743f2600-b4a7-4963-8bef-692428b6800a (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 22195cd0-81ca-4c14-8294-e0bb8767fa51, kind: Service { id: 743f2600-b4a7-4963-8bef-692428b6800a }, name: Name(\"nexus-743f2600-b4a7-4963-8bef-692428b6800a\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 167, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f71d33aa-7320-48b9-81d2-899a100e78cd (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 40c9a9db-bd0a-45df-b2ea-f8a9ad278c76 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: eb0487cd-afd0-473b-b56c-c84ebdb9a402 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 683f3bcd-05eb-48ca-a731-4591f3875c45 (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 100f148b-d0a5-4957-b7f4-7b142438883b (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 1ecfbd35-39cb-454a-b9bf-b046feb4f811 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: 014247db-8886-4cf5-b68e-375be0dabc7f (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 7361bad6-7bc4-4252-9897-84b3278d100e (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6e9d55e6-6648-4adf-a2be-39839524cd72 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ceaca86-5562-442f-a7a7-bbfd684c302b (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 771664a2-5c9b-4063-8f72-848056c94344 (service), filesystem_pool: Some(ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37091b30-13af-44d8-ac11-483eada5eeca (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 25c723ea-c52b-44b6-8751-dbe925d1e7a7 (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External } } } }, OmicronZoneConfig { id: f12a4089-4d9a-4d0c-b3e6-8a12d533bb03 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.072748Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.073097221Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.07310371Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.084855288Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.084887857Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.095575892Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.095605855Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.106755314Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.106785372Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.11726658Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.117295831Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.127832752Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.127867253Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.136945311Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.136975856Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.14623428Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.146265341Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.155362171Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.155388909Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.165729618Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.165763959Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.174722662Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.174759367Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.184531302Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.184565878Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.194978536Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.195007467Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.205097141Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.205128867Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.214937097Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.214963489Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.223901606Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.223932084Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.232929494Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.232958089Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.242063022Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.242093126Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.250973221Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.251000513Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.260592754Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.260621752Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.269663659Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.269712683Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.270042579Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.270164017Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:39.365634303Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:39.365690198Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:39.365697848Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:39.365702957Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:39.365710111Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:39.365715633Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:39.36572108Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:39.365728233Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:39.365733493Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:39.365738771Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:39.365743787Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:39.365748831Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:39.365819634Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:39.365838863Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:39.365845482Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:39.36585196Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:39.365857322Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:39.365862647Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:39.36586756Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.365958238Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"dd1af3d1-a804-4e95-af24-8a65814ce61d","remote_addr":"[fd00:1122:3344:101::1]:53729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b\", TimeNotSynchronized), (\"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b\", TimeNotSynchronized), (\"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca\", TimeNotSynchronized), (\"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a\", TimeNotSynchronized), (\"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7\", TimeNotSynchronized), (\"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811\", TimeNotSynchronized), (\"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd\", TimeNotSynchronized), (\"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\", TimeNotSynchronized), (\"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\", TimeNotSynchronized), (\"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344\", TimeNotSynchronized), (\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\", TimeNotSynchronized), (\"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a\", TimeNotSynchronized), (\"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402\", TimeNotSynchronized), (\"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f\", TimeNotSynchronized), (\"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e\", TimeNotSynchronized), (\"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d\", TimeNotSynchronized), (\"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301\", TimeNotSynchronized), (\"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45\", TimeNotSynchronized), (\"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\", TimeNotSynchronized)]","latency_us":292274,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:39.366126317Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"361.718671ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"dd1af3d1-a804-4e95-af24-8a65814ce61d\", \"content-length\": \"124\", \"date\": \"Wed, 15 Jan 2025 02:24:38 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"dd1af3d1-a804-4e95-af24-8a65814ce61d\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e94e881-2bab-4d68-8315-741d3cd1814a (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a4d0b75d-0c67-4224-af19-0510665c8775 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 44ac331a-da63-47e3-a636-214c3ca86b22 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 46c4b5d7-b839-4d27-ac2d-e1f6f6eae7d3, kind: Service { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 }, name: Name(\"ntp-fd5ec85c-1d4a-43a3-bd64-3bf287ecded3\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 9])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 77bd825f-cbb2-4b31-ba9f-d69e9e4cff59 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: a6c98ed9-62e0-43ae-9066-c1d4ee8e0596 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 788bf241-055b-45bf-8d67-e865c6324e46 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 82a65d0c-f391-4b1f-9c11-e3a397904f10 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 03e9536c-5029-447f-944a-4daff2909c7f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7468547b-b2f3-4993-afd8-72e9164a4301 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: b15378a9-5120-477b-abef-a170753103c1, kind: Service { id: 7468547b-b2f3-4993-afd8-72e9164a4301 }, name: Name(\"external-dns-7468547b-b2f3-4993-afd8-72e9164a4301\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 124])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beea2870-fa6b-4d58-b0eb-af5a33149d11, kind: Service { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a }, name: Name(\"external-dns-8b302e6b-b09c-4ef2-8768-1e5fae50d31a\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 41])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 357305ec-a785-4a6e-96cf-879dc3168076, kind: Service { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f }, name: Name(\"nexus-a1e38c68-2155-4ce9-b70a-66d434b3f24f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 213])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 56f43fd0-1c39-449d-9089-5f10b7acf59b, kind: Service { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d }, name: Name(\"nexus-b4bce764-527e-4a0d-9efa-46f5760fbb1d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 31])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 743f2600-b4a7-4963-8bef-692428b6800a (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 22195cd0-81ca-4c14-8294-e0bb8767fa51, kind: Service { id: 743f2600-b4a7-4963-8bef-692428b6800a }, name: Name(\"nexus-743f2600-b4a7-4963-8bef-692428b6800a\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 167, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f71d33aa-7320-48b9-81d2-899a100e78cd (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 40c9a9db-bd0a-45df-b2ea-f8a9ad278c76 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: eb0487cd-afd0-473b-b56c-c84ebdb9a402 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 683f3bcd-05eb-48ca-a731-4591f3875c45 (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 100f148b-d0a5-4957-b7f4-7b142438883b (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 1ecfbd35-39cb-454a-b9bf-b046feb4f811 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: 014247db-8886-4cf5-b68e-375be0dabc7f (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 7361bad6-7bc4-4252-9897-84b3278d100e (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6e9d55e6-6648-4adf-a2be-39839524cd72 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ceaca86-5562-442f-a7a7-bbfd684c302b (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 771664a2-5c9b-4063-8f72-848056c94344 (service), filesystem_pool: Some(ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37091b30-13af-44d8-ac11-483eada5eeca (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 25c723ea-c52b-44b6-8751-dbe925d1e7a7 (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External } } } }, OmicronZoneConfig { id: f12a4089-4d9a-4d0c-b3e6-8a12d533bb03 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.729548864Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.729894992Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.72990297Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.741243618Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.741291648Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.751743079Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.751781847Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.760537507Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.760573715Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.769637841Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.769689433Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.77855497Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.778593493Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.786777311Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.786820335Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.794808806Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.794850648Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.803126406Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.803162126Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.812295656Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.8123424Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.821259239Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.821301427Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.829558649Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.829597041Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.837568167Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.837608547Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.846403516Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.846437961Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.855354428Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.855402119Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.863650955Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.863687285Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.871704137Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.871743616Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.879888697Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.879927857Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.887860548Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.887899999Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.895952258Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.895997391Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.904800937Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.904847814Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.905047695Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.905173857Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:39.998671502Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:39.998708872Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:39.998714534Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:39.998718641Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:39.998723544Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:39.998727425Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:39.99873141Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:39.998735376Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:39.99873921Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:39.998743138Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:39.99876097Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:39.998766221Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:39.998770477Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:39.998775343Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:39.998779271Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:39.99878303Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:39.998787689Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:39.998791543Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:39.998796296Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:39.998872217Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"57e50da3-8d5c-447e-bfb3-c8ad56c516ff","remote_addr":"[fd00:1122:3344:101::1]:53729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e\", TimeNotSynchronized), (\"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b\", TimeNotSynchronized), (\"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a\", TimeNotSynchronized), (\"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f\", TimeNotSynchronized), (\"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca\", TimeNotSynchronized), (\"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\", TimeNotSynchronized), (\"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\", TimeNotSynchronized), (\"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a\", TimeNotSynchronized), (\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\", TimeNotSynchronized), (\"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45\", TimeNotSynchronized), (\"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301\", TimeNotSynchronized), (\"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b\", TimeNotSynchronized), (\"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811\", TimeNotSynchronized), (\"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344\", TimeNotSynchronized), (\"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7\", TimeNotSynchronized), (\"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\", TimeNotSynchronized), (\"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d\", TimeNotSynchronized), (\"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402\", TimeNotSynchronized), (\"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd\", TimeNotSynchronized)]","latency_us":268455,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:39.999008252Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"225.906242ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"57e50da3-8d5c-447e-bfb3-c8ad56c516ff\", \"content-length\": \"124\", \"date\": \"Wed, 15 Jan 2025 02:24:39 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"57e50da3-8d5c-447e-bfb3-c8ad56c516ff\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e94e881-2bab-4d68-8315-741d3cd1814a (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a4d0b75d-0c67-4224-af19-0510665c8775 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 44ac331a-da63-47e3-a636-214c3ca86b22 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 46c4b5d7-b839-4d27-ac2d-e1f6f6eae7d3, kind: Service { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 }, name: Name(\"ntp-fd5ec85c-1d4a-43a3-bd64-3bf287ecded3\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 9])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 77bd825f-cbb2-4b31-ba9f-d69e9e4cff59 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: a6c98ed9-62e0-43ae-9066-c1d4ee8e0596 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 788bf241-055b-45bf-8d67-e865c6324e46 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 82a65d0c-f391-4b1f-9c11-e3a397904f10 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 03e9536c-5029-447f-944a-4daff2909c7f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7468547b-b2f3-4993-afd8-72e9164a4301 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: b15378a9-5120-477b-abef-a170753103c1, kind: Service { id: 7468547b-b2f3-4993-afd8-72e9164a4301 }, name: Name(\"external-dns-7468547b-b2f3-4993-afd8-72e9164a4301\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 124])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beea2870-fa6b-4d58-b0eb-af5a33149d11, kind: Service { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a }, name: Name(\"external-dns-8b302e6b-b09c-4ef2-8768-1e5fae50d31a\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 41])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 357305ec-a785-4a6e-96cf-879dc3168076, kind: Service { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f }, name: Name(\"nexus-a1e38c68-2155-4ce9-b70a-66d434b3f24f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 213])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 56f43fd0-1c39-449d-9089-5f10b7acf59b, kind: Service { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d }, name: Name(\"nexus-b4bce764-527e-4a0d-9efa-46f5760fbb1d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 31])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 743f2600-b4a7-4963-8bef-692428b6800a (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 22195cd0-81ca-4c14-8294-e0bb8767fa51, kind: Service { id: 743f2600-b4a7-4963-8bef-692428b6800a }, name: Name(\"nexus-743f2600-b4a7-4963-8bef-692428b6800a\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 167, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f71d33aa-7320-48b9-81d2-899a100e78cd (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 40c9a9db-bd0a-45df-b2ea-f8a9ad278c76 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: eb0487cd-afd0-473b-b56c-c84ebdb9a402 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 683f3bcd-05eb-48ca-a731-4591f3875c45 (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 100f148b-d0a5-4957-b7f4-7b142438883b (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 1ecfbd35-39cb-454a-b9bf-b046feb4f811 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: 014247db-8886-4cf5-b68e-375be0dabc7f (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 7361bad6-7bc4-4252-9897-84b3278d100e (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6e9d55e6-6648-4adf-a2be-39839524cd72 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ceaca86-5562-442f-a7a7-bbfd684c302b (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 771664a2-5c9b-4063-8f72-848056c94344 (service), filesystem_pool: Some(ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37091b30-13af-44d8-ac11-483eada5eeca (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 25c723ea-c52b-44b6-8751-dbe925d1e7a7 (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External } } } }, OmicronZoneConfig { id: f12a4089-4d9a-4d0c-b3e6-8a12d533bb03 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.227020383Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.227324939Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.227332533Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.238321332Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.238356452Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.24876401Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.24878832Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.25901113Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.259040306Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.268542244Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.268574185Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.278823378Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.278851147Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.287905133Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.287932518Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.297831375Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.297860363Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.307027947Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.30706648Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.316484468Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.316514562Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.326110825Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.326139148Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.335561692Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.335594712Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.344765674Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.344798159Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.354806209Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.354842266Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.36388729Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.363917497Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.373368025Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.373395504Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.382647485Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.382674195Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.391807288Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.391835507Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.401107411Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.401146431Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.410506502Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.410536381Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.420447449Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.420488128Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.420757953Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.420867334Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:40.518374745Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:40.518429356Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:40.518434043Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:40.518437654Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:40.518441187Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:40.518444779Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:40.518448379Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:40.518458016Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:40.518461747Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:40.518465207Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:40.518469041Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:40.518472717Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:40.518560178Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:40.518568597Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:40.518573697Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:40.51858147Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:40.518587057Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:40.518593826Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:40.518602967Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.51872372Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"bbddb148-028f-49f2-96cc-0a1361284365","remote_addr":"[fd00:1122:3344:101::1]:53729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b\", TimeNotSynchronized), (\"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\", TimeNotSynchronized), (\"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402\", TimeNotSynchronized), (\"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45\", TimeNotSynchronized), (\"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca\", TimeNotSynchronized), (\"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e\", TimeNotSynchronized), (\"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7\", TimeNotSynchronized), (\"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a\", TimeNotSynchronized), (\"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d\", TimeNotSynchronized), (\"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b\", TimeNotSynchronized), (\"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811\", TimeNotSynchronized), (\"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\", TimeNotSynchronized), (\"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\", TimeNotSynchronized), (\"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd\", TimeNotSynchronized), (\"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344\", TimeNotSynchronized), (\"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f\", TimeNotSynchronized), (\"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a\", TimeNotSynchronized), (\"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301\", TimeNotSynchronized), (\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\", TimeNotSynchronized)]","latency_us":290813,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:40.518941553Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"430.948693ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"bbddb148-028f-49f2-96cc-0a1361284365\", \"content-length\": \"124\", \"date\": \"Wed, 15 Jan 2025 02:24:39 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"bbddb148-028f-49f2-96cc-0a1361284365\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e94e881-2bab-4d68-8315-741d3cd1814a (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a4d0b75d-0c67-4224-af19-0510665c8775 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 44ac331a-da63-47e3-a636-214c3ca86b22 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 46c4b5d7-b839-4d27-ac2d-e1f6f6eae7d3, kind: Service { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 }, name: Name(\"ntp-fd5ec85c-1d4a-43a3-bd64-3bf287ecded3\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 9])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 77bd825f-cbb2-4b31-ba9f-d69e9e4cff59 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: a6c98ed9-62e0-43ae-9066-c1d4ee8e0596 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 788bf241-055b-45bf-8d67-e865c6324e46 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 82a65d0c-f391-4b1f-9c11-e3a397904f10 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 03e9536c-5029-447f-944a-4daff2909c7f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7468547b-b2f3-4993-afd8-72e9164a4301 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: b15378a9-5120-477b-abef-a170753103c1, kind: Service { id: 7468547b-b2f3-4993-afd8-72e9164a4301 }, name: Name(\"external-dns-7468547b-b2f3-4993-afd8-72e9164a4301\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 124])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beea2870-fa6b-4d58-b0eb-af5a33149d11, kind: Service { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a }, name: Name(\"external-dns-8b302e6b-b09c-4ef2-8768-1e5fae50d31a\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 41])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 357305ec-a785-4a6e-96cf-879dc3168076, kind: Service { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f }, name: Name(\"nexus-a1e38c68-2155-4ce9-b70a-66d434b3f24f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 213])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 56f43fd0-1c39-449d-9089-5f10b7acf59b, kind: Service { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d }, name: Name(\"nexus-b4bce764-527e-4a0d-9efa-46f5760fbb1d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 31])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 743f2600-b4a7-4963-8bef-692428b6800a (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 22195cd0-81ca-4c14-8294-e0bb8767fa51, kind: Service { id: 743f2600-b4a7-4963-8bef-692428b6800a }, name: Name(\"nexus-743f2600-b4a7-4963-8bef-692428b6800a\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 167, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f71d33aa-7320-48b9-81d2-899a100e78cd (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 40c9a9db-bd0a-45df-b2ea-f8a9ad278c76 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: eb0487cd-afd0-473b-b56c-c84ebdb9a402 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 683f3bcd-05eb-48ca-a731-4591f3875c45 (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 100f148b-d0a5-4957-b7f4-7b142438883b (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 1ecfbd35-39cb-454a-b9bf-b046feb4f811 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: 014247db-8886-4cf5-b68e-375be0dabc7f (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 7361bad6-7bc4-4252-9897-84b3278d100e (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6e9d55e6-6648-4adf-a2be-39839524cd72 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ceaca86-5562-442f-a7a7-bbfd684c302b (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 771664a2-5c9b-4063-8f72-848056c94344 (service), filesystem_pool: Some(ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37091b30-13af-44d8-ac11-483eada5eeca (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 25c723ea-c52b-44b6-8751-dbe925d1e7a7 (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External } } } }, OmicronZoneConfig { id: f12a4089-4d9a-4d0c-b3e6-8a12d533bb03 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.951602869Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.951878895Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.951886902Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.96296498Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.963017246Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.972462772Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.972520721Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.981045515Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.981089635Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.989742888Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.989787889Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.997781818Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:40.997821588Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.005830458Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.005872731Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.01378984Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.013830941Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.021680335Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.021721277Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.030006518Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.03004071Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.038203672Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.038244341Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.046388918Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.046436067Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.054447484Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.054495702Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.062579109Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.062701513Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.07079581Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.070835674Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.07883458Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.078873797Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.08680001Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.086841505Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.094828198Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.094869271Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.102903124Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.10293949Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.11117054Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.111210385Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.119195769Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.119243508Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.119440266Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.119645157Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:41.209222155Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:41.209259628Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:41.209265609Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:41.209272679Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:41.20929441Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:41.209301227Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:41.209306261Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:41.209311333Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:41.209316321Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:41.209320961Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:41.209325761Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:41.209330524Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:41.209335372Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:41.209341475Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:41.2093465Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:41.209350147Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:41.2093549Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:41.209358312Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:41.209361753Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.20943052Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"a130d4c3-dce7-48c1-874a-10ff117966c9","remote_addr":"[fd00:1122:3344:101::1]:53729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301\", TimeNotSynchronized), (\"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca\", TimeNotSynchronized), (\"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\", TimeNotSynchronized), (\"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a\", TimeNotSynchronized), (\"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\", TimeNotSynchronized), (\"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45\", TimeNotSynchronized), (\"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e\", TimeNotSynchronized), (\"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b\", TimeNotSynchronized), (\"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\", TimeNotSynchronized), (\"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d\", TimeNotSynchronized), (\"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344\", TimeNotSynchronized), (\"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a\", TimeNotSynchronized), (\"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f\", TimeNotSynchronized), (\"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd\", TimeNotSynchronized), (\"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811\", TimeNotSynchronized), (\"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b\", TimeNotSynchronized), (\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\", TimeNotSynchronized), (\"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7\", TimeNotSynchronized), (\"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402\", TimeNotSynchronized)]","latency_us":257021,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:41.209598403Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"568.962828ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a130d4c3-dce7-48c1-874a-10ff117966c9\", \"content-length\": \"124\", \"date\": \"Wed, 15 Jan 2025 02:24:41 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a130d4c3-dce7-48c1-874a-10ff117966c9\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e94e881-2bab-4d68-8315-741d3cd1814a (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a4d0b75d-0c67-4224-af19-0510665c8775 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 44ac331a-da63-47e3-a636-214c3ca86b22 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 46c4b5d7-b839-4d27-ac2d-e1f6f6eae7d3, kind: Service { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 }, name: Name(\"ntp-fd5ec85c-1d4a-43a3-bd64-3bf287ecded3\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 9])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 77bd825f-cbb2-4b31-ba9f-d69e9e4cff59 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: a6c98ed9-62e0-43ae-9066-c1d4ee8e0596 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 788bf241-055b-45bf-8d67-e865c6324e46 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 82a65d0c-f391-4b1f-9c11-e3a397904f10 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 03e9536c-5029-447f-944a-4daff2909c7f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7468547b-b2f3-4993-afd8-72e9164a4301 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: b15378a9-5120-477b-abef-a170753103c1, kind: Service { id: 7468547b-b2f3-4993-afd8-72e9164a4301 }, name: Name(\"external-dns-7468547b-b2f3-4993-afd8-72e9164a4301\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 124])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beea2870-fa6b-4d58-b0eb-af5a33149d11, kind: Service { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a }, name: Name(\"external-dns-8b302e6b-b09c-4ef2-8768-1e5fae50d31a\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 41])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 357305ec-a785-4a6e-96cf-879dc3168076, kind: Service { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f }, name: Name(\"nexus-a1e38c68-2155-4ce9-b70a-66d434b3f24f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 213])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 56f43fd0-1c39-449d-9089-5f10b7acf59b, kind: Service { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d }, name: Name(\"nexus-b4bce764-527e-4a0d-9efa-46f5760fbb1d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 31])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 743f2600-b4a7-4963-8bef-692428b6800a (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 22195cd0-81ca-4c14-8294-e0bb8767fa51, kind: Service { id: 743f2600-b4a7-4963-8bef-692428b6800a }, name: Name(\"nexus-743f2600-b4a7-4963-8bef-692428b6800a\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 167, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f71d33aa-7320-48b9-81d2-899a100e78cd (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 40c9a9db-bd0a-45df-b2ea-f8a9ad278c76 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: eb0487cd-afd0-473b-b56c-c84ebdb9a402 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 683f3bcd-05eb-48ca-a731-4591f3875c45 (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 100f148b-d0a5-4957-b7f4-7b142438883b (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 1ecfbd35-39cb-454a-b9bf-b046feb4f811 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: 014247db-8886-4cf5-b68e-375be0dabc7f (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 7361bad6-7bc4-4252-9897-84b3278d100e (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6e9d55e6-6648-4adf-a2be-39839524cd72 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ceaca86-5562-442f-a7a7-bbfd684c302b (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 771664a2-5c9b-4063-8f72-848056c94344 (service), filesystem_pool: Some(ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37091b30-13af-44d8-ac11-483eada5eeca (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 25c723ea-c52b-44b6-8751-dbe925d1e7a7 (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External } } } }, OmicronZoneConfig { id: f12a4089-4d9a-4d0c-b3e6-8a12d533bb03 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.781087572Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.781542481Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.781552944Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.792554423Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.792598384Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.801990214Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.802026299Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.813052508Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.81308594Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.82282717Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.822856421Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.831856958Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.831890249Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.8411457Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.841204418Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.849983703Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.850013629Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.860229858Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.860272243Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.869969519Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.87000358Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.879389156Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.879426143Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.888543594Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.888655522Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.89936252Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.899400256Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.908712658Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.90875812Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.918210128Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.918247263Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.927634121Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.927739271Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.936602476Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.93664787Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.945584362Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.945619951Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.956183042Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.95621625Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.965717277Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.965749715Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.975064111Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.975111183Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.975314692Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:41.975421776Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:42.072462732Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:42.072532128Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:42.072542187Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:42.07254904Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:42.072555688Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:42.072563676Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:42.072572657Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:42.072579932Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:42.072587929Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:42.072594042Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:42.072600492Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:42.072607748Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:42.072615276Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:42.072623086Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:42.072627709Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:42.072633727Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:42.072652216Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:42.07265815Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:42.072664478Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:42.072710436Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"d204aa66-0b84-41b6-ac4f-14a19d1da721","remote_addr":"[fd00:1122:3344:101::1]:53729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402\", TimeNotSynchronized), (\"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b\", TimeNotSynchronized), (\"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344\", TimeNotSynchronized), (\"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca\", TimeNotSynchronized), (\"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45\", TimeNotSynchronized), (\"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7\", TimeNotSynchronized), (\"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\", TimeNotSynchronized), (\"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811\", TimeNotSynchronized), (\"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\", TimeNotSynchronized), (\"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d\", TimeNotSynchronized), (\"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\", TimeNotSynchronized), (\"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301\", TimeNotSynchronized), (\"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e\", TimeNotSynchronized), (\"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b\", TimeNotSynchronized), (\"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f\", TimeNotSynchronized), (\"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a\", TimeNotSynchronized), (\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\", TimeNotSynchronized), (\"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a\", TimeNotSynchronized), (\"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd\", TimeNotSynchronized)]","latency_us":290586,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:42.072874278Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"491.030242ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d204aa66-0b84-41b6-ac4f-14a19d1da721\", \"content-length\": \"124\", \"date\": \"Wed, 15 Jan 2025 02:24:42 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d204aa66-0b84-41b6-ac4f-14a19d1da721\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e94e881-2bab-4d68-8315-741d3cd1814a (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a4d0b75d-0c67-4224-af19-0510665c8775 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 44ac331a-da63-47e3-a636-214c3ca86b22 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 46c4b5d7-b839-4d27-ac2d-e1f6f6eae7d3, kind: Service { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 }, name: Name(\"ntp-fd5ec85c-1d4a-43a3-bd64-3bf287ecded3\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 9])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 77bd825f-cbb2-4b31-ba9f-d69e9e4cff59 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: a6c98ed9-62e0-43ae-9066-c1d4ee8e0596 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 788bf241-055b-45bf-8d67-e865c6324e46 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 82a65d0c-f391-4b1f-9c11-e3a397904f10 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 03e9536c-5029-447f-944a-4daff2909c7f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7468547b-b2f3-4993-afd8-72e9164a4301 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: b15378a9-5120-477b-abef-a170753103c1, kind: Service { id: 7468547b-b2f3-4993-afd8-72e9164a4301 }, name: Name(\"external-dns-7468547b-b2f3-4993-afd8-72e9164a4301\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 124])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beea2870-fa6b-4d58-b0eb-af5a33149d11, kind: Service { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a }, name: Name(\"external-dns-8b302e6b-b09c-4ef2-8768-1e5fae50d31a\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 41])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 357305ec-a785-4a6e-96cf-879dc3168076, kind: Service { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f }, name: Name(\"nexus-a1e38c68-2155-4ce9-b70a-66d434b3f24f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 213])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 56f43fd0-1c39-449d-9089-5f10b7acf59b, kind: Service { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d }, name: Name(\"nexus-b4bce764-527e-4a0d-9efa-46f5760fbb1d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 31])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 743f2600-b4a7-4963-8bef-692428b6800a (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 22195cd0-81ca-4c14-8294-e0bb8767fa51, kind: Service { id: 743f2600-b4a7-4963-8bef-692428b6800a }, name: Name(\"nexus-743f2600-b4a7-4963-8bef-692428b6800a\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 167, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f71d33aa-7320-48b9-81d2-899a100e78cd (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 40c9a9db-bd0a-45df-b2ea-f8a9ad278c76 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: eb0487cd-afd0-473b-b56c-c84ebdb9a402 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 683f3bcd-05eb-48ca-a731-4591f3875c45 (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 100f148b-d0a5-4957-b7f4-7b142438883b (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 1ecfbd35-39cb-454a-b9bf-b046feb4f811 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: 014247db-8886-4cf5-b68e-375be0dabc7f (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 7361bad6-7bc4-4252-9897-84b3278d100e (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6e9d55e6-6648-4adf-a2be-39839524cd72 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ceaca86-5562-442f-a7a7-bbfd684c302b (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 771664a2-5c9b-4063-8f72-848056c94344 (service), filesystem_pool: Some(ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37091b30-13af-44d8-ac11-483eada5eeca (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 25c723ea-c52b-44b6-8751-dbe925d1e7a7 (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External } } } }, OmicronZoneConfig { id: f12a4089-4d9a-4d0c-b3e6-8a12d533bb03 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:42.566155842Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:42.566797711Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:42.56682159Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:42.578392578Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:42.578498188Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:42.589189818Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:42.589225801Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:42.599451235Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:42.599549335Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:42.608685428Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:42.608719076Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:42.618412676Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:42.618448181Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:42.62762294Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:42.627659062Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:42.636732341Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:42.636764432Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:42.647197131Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:42.647231622Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:42.657328522Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:42.657360276Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:42.667302456Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:42.667344419Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:42.67723931Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:42.677270989Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:42.687139675Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:42.687178226Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:42.697256762Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:42.697290353Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:42.707193359Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:42.707225666Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:42.716929326Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:42.716972321Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:42.727124551Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:42.727163533Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:42.736194799Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:42.736229544Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:42.747147741Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:42.747190079Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:42.757162056Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:42.757196557Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:42.766605145Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:42.766650194Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:42.766847749Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:42.767044778Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:42.866036021Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:42.86608782Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:42.866096567Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:42.866101255Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:42.866107161Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:42.866111399Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:42.866116818Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:42.866121205Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:42.866128602Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:42.866134405Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:42.866139937Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:42.86614589Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:42.8661516Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:42.866158819Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:42.866173116Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:42.866177578Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:42.866246478Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:42.866252451Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:42.86625652Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:42.866319005Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"f451b0b1-26ec-40dc-8146-55b5b306805f","remote_addr":"[fd00:1122:3344:101::1]:53729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd\", TimeNotSynchronized), (\"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\", TimeNotSynchronized), (\"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e\", TimeNotSynchronized), (\"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7\", TimeNotSynchronized), (\"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301\", TimeNotSynchronized), (\"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a\", TimeNotSynchronized), (\"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f\", TimeNotSynchronized), (\"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402\", TimeNotSynchronized), (\"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a\", TimeNotSynchronized), (\"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b\", TimeNotSynchronized), (\"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b\", TimeNotSynchronized), (\"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344\", TimeNotSynchronized), (\"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca\", TimeNotSynchronized), (\"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d\", TimeNotSynchronized), (\"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\", TimeNotSynchronized), (\"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811\", TimeNotSynchronized), (\"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\", TimeNotSynchronized), (\"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45\", TimeNotSynchronized), (\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\", TimeNotSynchronized)]","latency_us":299062,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:42.866477222Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"679.291721ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f451b0b1-26ec-40dc-8146-55b5b306805f\", \"content-length\": \"124\", \"date\": \"Wed, 15 Jan 2025 02:24:42 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f451b0b1-26ec-40dc-8146-55b5b306805f\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e94e881-2bab-4d68-8315-741d3cd1814a (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a4d0b75d-0c67-4224-af19-0510665c8775 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 44ac331a-da63-47e3-a636-214c3ca86b22 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 46c4b5d7-b839-4d27-ac2d-e1f6f6eae7d3, kind: Service { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 }, name: Name(\"ntp-fd5ec85c-1d4a-43a3-bd64-3bf287ecded3\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 9])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 77bd825f-cbb2-4b31-ba9f-d69e9e4cff59 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: a6c98ed9-62e0-43ae-9066-c1d4ee8e0596 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 788bf241-055b-45bf-8d67-e865c6324e46 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 82a65d0c-f391-4b1f-9c11-e3a397904f10 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 03e9536c-5029-447f-944a-4daff2909c7f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7468547b-b2f3-4993-afd8-72e9164a4301 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: b15378a9-5120-477b-abef-a170753103c1, kind: Service { id: 7468547b-b2f3-4993-afd8-72e9164a4301 }, name: Name(\"external-dns-7468547b-b2f3-4993-afd8-72e9164a4301\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 124])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beea2870-fa6b-4d58-b0eb-af5a33149d11, kind: Service { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a }, name: Name(\"external-dns-8b302e6b-b09c-4ef2-8768-1e5fae50d31a\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 41])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 357305ec-a785-4a6e-96cf-879dc3168076, kind: Service { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f }, name: Name(\"nexus-a1e38c68-2155-4ce9-b70a-66d434b3f24f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 213])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 56f43fd0-1c39-449d-9089-5f10b7acf59b, kind: Service { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d }, name: Name(\"nexus-b4bce764-527e-4a0d-9efa-46f5760fbb1d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 31])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 743f2600-b4a7-4963-8bef-692428b6800a (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 22195cd0-81ca-4c14-8294-e0bb8767fa51, kind: Service { id: 743f2600-b4a7-4963-8bef-692428b6800a }, name: Name(\"nexus-743f2600-b4a7-4963-8bef-692428b6800a\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 167, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f71d33aa-7320-48b9-81d2-899a100e78cd (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 40c9a9db-bd0a-45df-b2ea-f8a9ad278c76 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: eb0487cd-afd0-473b-b56c-c84ebdb9a402 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 683f3bcd-05eb-48ca-a731-4591f3875c45 (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 100f148b-d0a5-4957-b7f4-7b142438883b (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 1ecfbd35-39cb-454a-b9bf-b046feb4f811 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: 014247db-8886-4cf5-b68e-375be0dabc7f (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 7361bad6-7bc4-4252-9897-84b3278d100e (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6e9d55e6-6648-4adf-a2be-39839524cd72 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ceaca86-5562-442f-a7a7-bbfd684c302b (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 771664a2-5c9b-4063-8f72-848056c94344 (service), filesystem_pool: Some(ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37091b30-13af-44d8-ac11-483eada5eeca (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 25c723ea-c52b-44b6-8751-dbe925d1e7a7 (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External } } } }, OmicronZoneConfig { id: f12a4089-4d9a-4d0c-b3e6-8a12d533bb03 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:43.548856112Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:43.549207672Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:43.549213822Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:43.559963682Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:43.560008102Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:43.570300268Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:43.57034Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:43.581402089Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:43.581435211Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:43.592323168Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:43.592367044Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:43.603064806Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:43.603096999Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:43.612543979Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:43.612575255Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:43.621863667Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:43.621894765Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:43.630699406Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:43.630732369Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:43.638901537Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:43.638942582Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:43.64737334Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:43.647407841Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:43.655790189Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:43.65583444Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:43.665480258Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:43.665519765Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:43.674546164Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:43.674582081Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:43.683004142Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:43.683050447Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:43.692144959Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:43.692185066Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:43.70207006Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:43.702106333Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:43.711604175Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:43.71164762Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:43.720756104Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:43.720798611Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:43.729691905Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:43.729730381Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:43.739276707Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:43.739323424Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:43.739606063Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:43.739717366Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:43.83376486Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:43.833866598Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:43.833872269Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:43.833876742Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:43.833880802Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:43.833884983Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:43.833889192Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:43.8339602Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:43.833968188Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:43.833974226Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:43.833980076Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:43.833999876Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:43.834004779Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:43.834009711Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:43.834014858Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:43.834020933Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:43.834026043Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:43.834030758Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:43.834035699Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:43.834123771Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"5f71ef96-3816-461f-b9ae-e6dbbe3b4aca","remote_addr":"[fd00:1122:3344:101::1]:53729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f\", TimeNotSynchronized), (\"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b\", TimeNotSynchronized), (\"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301\", TimeNotSynchronized), (\"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a\", TimeNotSynchronized), (\"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd\", TimeNotSynchronized), (\"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\", TimeNotSynchronized), (\"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e\", TimeNotSynchronized), (\"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\", TimeNotSynchronized), (\"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a\", TimeNotSynchronized), (\"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b\", TimeNotSynchronized), (\"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344\", TimeNotSynchronized), (\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\", TimeNotSynchronized), (\"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca\", TimeNotSynchronized), (\"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7\", TimeNotSynchronized), (\"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811\", TimeNotSynchronized), (\"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402\", TimeNotSynchronized), (\"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\", TimeNotSynchronized), (\"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d\", TimeNotSynchronized), (\"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45\", TimeNotSynchronized)]","latency_us":284350,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:43.834269668Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"950.876718ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5f71ef96-3816-461f-b9ae-e6dbbe3b4aca\", \"content-length\": \"124\", \"date\": \"Wed, 15 Jan 2025 02:24:43 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5f71ef96-3816-461f-b9ae-e6dbbe3b4aca\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e94e881-2bab-4d68-8315-741d3cd1814a (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a4d0b75d-0c67-4224-af19-0510665c8775 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 44ac331a-da63-47e3-a636-214c3ca86b22 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 46c4b5d7-b839-4d27-ac2d-e1f6f6eae7d3, kind: Service { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 }, name: Name(\"ntp-fd5ec85c-1d4a-43a3-bd64-3bf287ecded3\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 9])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 77bd825f-cbb2-4b31-ba9f-d69e9e4cff59 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: a6c98ed9-62e0-43ae-9066-c1d4ee8e0596 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 788bf241-055b-45bf-8d67-e865c6324e46 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 82a65d0c-f391-4b1f-9c11-e3a397904f10 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 03e9536c-5029-447f-944a-4daff2909c7f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7468547b-b2f3-4993-afd8-72e9164a4301 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: b15378a9-5120-477b-abef-a170753103c1, kind: Service { id: 7468547b-b2f3-4993-afd8-72e9164a4301 }, name: Name(\"external-dns-7468547b-b2f3-4993-afd8-72e9164a4301\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 124])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beea2870-fa6b-4d58-b0eb-af5a33149d11, kind: Service { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a }, name: Name(\"external-dns-8b302e6b-b09c-4ef2-8768-1e5fae50d31a\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 41])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 357305ec-a785-4a6e-96cf-879dc3168076, kind: Service { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f }, name: Name(\"nexus-a1e38c68-2155-4ce9-b70a-66d434b3f24f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 213])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 56f43fd0-1c39-449d-9089-5f10b7acf59b, kind: Service { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d }, name: Name(\"nexus-b4bce764-527e-4a0d-9efa-46f5760fbb1d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 31])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 743f2600-b4a7-4963-8bef-692428b6800a (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 22195cd0-81ca-4c14-8294-e0bb8767fa51, kind: Service { id: 743f2600-b4a7-4963-8bef-692428b6800a }, name: Name(\"nexus-743f2600-b4a7-4963-8bef-692428b6800a\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 167, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f71d33aa-7320-48b9-81d2-899a100e78cd (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 40c9a9db-bd0a-45df-b2ea-f8a9ad278c76 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: eb0487cd-afd0-473b-b56c-c84ebdb9a402 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 683f3bcd-05eb-48ca-a731-4591f3875c45 (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 100f148b-d0a5-4957-b7f4-7b142438883b (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 1ecfbd35-39cb-454a-b9bf-b046feb4f811 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: 014247db-8886-4cf5-b68e-375be0dabc7f (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 7361bad6-7bc4-4252-9897-84b3278d100e (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6e9d55e6-6648-4adf-a2be-39839524cd72 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ceaca86-5562-442f-a7a7-bbfd684c302b (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 771664a2-5c9b-4063-8f72-848056c94344 (service), filesystem_pool: Some(ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37091b30-13af-44d8-ac11-483eada5eeca (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 25c723ea-c52b-44b6-8751-dbe925d1e7a7 (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External } } } }, OmicronZoneConfig { id: f12a4089-4d9a-4d0c-b3e6-8a12d533bb03 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:44.789550295Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:44.78983009Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:44.789838227Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:44.801924457Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:44.801966955Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:44.811919064Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:44.811962303Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:44.822259933Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:44.822301587Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:44.832628656Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:44.832665914Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:44.843247471Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:44.843278747Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:44.852871289Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:44.85290294Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:44.861822952Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:44.861856525Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:44.871871511Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:44.8719173Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:44.882602417Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:44.882701288Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:44.891919075Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:44.891958292Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:44.901891045Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:44.901930637Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:44.911230762Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:44.9112692Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:44.920366141Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:44.920414649Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:44.930929754Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:44.930964001Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:44.940855437Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:44.94089548Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:44.950040432Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:44.950082545Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:44.961774338Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:44.961807723Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:44.971032886Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:44.971066806Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:44.980735752Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:44.980772681Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:44.989764022Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:44.989803586Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:44.990096481Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:44.990213755Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:45.089239535Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:45.089291632Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:45.089297549Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:45.089304271Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:45.089309661Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:45.089316027Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:45.089321624Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:45.089327249Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:45.089342184Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:45.089347275Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:45.089352253Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:45.089355759Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:45.089360644Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:45.089364009Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:45.089368884Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:45.08937272Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:45.089377745Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:45.089382582Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:45.089387213Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:45.08956269Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"5e52b179-0265-4f63-9ad7-92d0716f6e42","remote_addr":"[fd00:1122:3344:101::1]:53729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b\", TimeNotSynchronized), (\"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca\", TimeNotSynchronized), (\"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a\", TimeNotSynchronized), (\"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811\", TimeNotSynchronized), (\"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\", TimeNotSynchronized), (\"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7\", TimeNotSynchronized), (\"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301\", TimeNotSynchronized), (\"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f\", TimeNotSynchronized), (\"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402\", TimeNotSynchronized), (\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\", TimeNotSynchronized), (\"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45\", TimeNotSynchronized), (\"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\", TimeNotSynchronized), (\"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344\", TimeNotSynchronized), (\"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\", TimeNotSynchronized), (\"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a\", TimeNotSynchronized), (\"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d\", TimeNotSynchronized), (\"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd\", TimeNotSynchronized), (\"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b\", TimeNotSynchronized), (\"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e\", TimeNotSynchronized)]","latency_us":298967,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:45.089721516Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"913.221774ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5e52b179-0265-4f63-9ad7-92d0716f6e42\", \"content-length\": \"124\", \"date\": \"Wed, 15 Jan 2025 02:24:45 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5e52b179-0265-4f63-9ad7-92d0716f6e42\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e94e881-2bab-4d68-8315-741d3cd1814a (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a4d0b75d-0c67-4224-af19-0510665c8775 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 44ac331a-da63-47e3-a636-214c3ca86b22 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 46c4b5d7-b839-4d27-ac2d-e1f6f6eae7d3, kind: Service { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 }, name: Name(\"ntp-fd5ec85c-1d4a-43a3-bd64-3bf287ecded3\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 9])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 77bd825f-cbb2-4b31-ba9f-d69e9e4cff59 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: a6c98ed9-62e0-43ae-9066-c1d4ee8e0596 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 788bf241-055b-45bf-8d67-e865c6324e46 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 82a65d0c-f391-4b1f-9c11-e3a397904f10 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 03e9536c-5029-447f-944a-4daff2909c7f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7468547b-b2f3-4993-afd8-72e9164a4301 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: b15378a9-5120-477b-abef-a170753103c1, kind: Service { id: 7468547b-b2f3-4993-afd8-72e9164a4301 }, name: Name(\"external-dns-7468547b-b2f3-4993-afd8-72e9164a4301\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 124])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beea2870-fa6b-4d58-b0eb-af5a33149d11, kind: Service { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a }, name: Name(\"external-dns-8b302e6b-b09c-4ef2-8768-1e5fae50d31a\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 41])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 357305ec-a785-4a6e-96cf-879dc3168076, kind: Service { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f }, name: Name(\"nexus-a1e38c68-2155-4ce9-b70a-66d434b3f24f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 213])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 56f43fd0-1c39-449d-9089-5f10b7acf59b, kind: Service { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d }, name: Name(\"nexus-b4bce764-527e-4a0d-9efa-46f5760fbb1d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 31])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 743f2600-b4a7-4963-8bef-692428b6800a (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 22195cd0-81ca-4c14-8294-e0bb8767fa51, kind: Service { id: 743f2600-b4a7-4963-8bef-692428b6800a }, name: Name(\"nexus-743f2600-b4a7-4963-8bef-692428b6800a\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 167, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f71d33aa-7320-48b9-81d2-899a100e78cd (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 40c9a9db-bd0a-45df-b2ea-f8a9ad278c76 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: eb0487cd-afd0-473b-b56c-c84ebdb9a402 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 683f3bcd-05eb-48ca-a731-4591f3875c45 (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 100f148b-d0a5-4957-b7f4-7b142438883b (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 1ecfbd35-39cb-454a-b9bf-b046feb4f811 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: 014247db-8886-4cf5-b68e-375be0dabc7f (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 7361bad6-7bc4-4252-9897-84b3278d100e (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6e9d55e6-6648-4adf-a2be-39839524cd72 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ceaca86-5562-442f-a7a7-bbfd684c302b (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 771664a2-5c9b-4063-8f72-848056c94344 (service), filesystem_pool: Some(ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37091b30-13af-44d8-ac11-483eada5eeca (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 25c723ea-c52b-44b6-8751-dbe925d1e7a7 (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External } } } }, OmicronZoneConfig { id: f12a4089-4d9a-4d0c-b3e6-8a12d533bb03 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:46.006628397Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:46.006967384Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:46.006977285Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:46.018206729Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:46.018259493Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:46.027226317Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:46.027284565Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:46.03750182Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:46.037543174Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:46.046423441Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:46.046469154Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:46.057297981Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:46.05733867Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:46.067707905Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:46.06774805Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:46.077554891Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:46.077603249Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:46.086750482Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:46.086790053Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:46.09713329Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:46.097172104Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:46.106203138Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:46.106240583Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:46.115098324Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:46.115136003Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:46.123347206Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:46.123392517Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:46.133847858Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:46.133886896Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:46.142667709Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:46.14271076Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:46.152783032Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:46.152822736Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:46.163802399Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:46.163839394Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:46.173037517Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:46.173070424Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:46.183967116Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:46.184011639Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:46.195663493Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:46.195701313Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:46.2058263Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:46.205882439Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:46.206094873Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:46.206306483Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:46.303720394Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:46.303758543Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:46.303763587Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:46.303767721Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:46.303771799Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:46.303791421Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:46.303795425Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:46.303798425Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:46.303802353Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:46.303806375Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:46.303810153Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:46.303813866Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:46.303816828Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:46.303819809Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:46.303823578Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:46.303826522Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:46.303829428Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:46.303833245Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:46.303836976Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:46.303892336Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"a05e08b7-edf5-4466-84a4-8a97a0031978","remote_addr":"[fd00:1122:3344:101::1]:53729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a\", TimeNotSynchronized), (\"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\", TimeNotSynchronized), (\"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca\", TimeNotSynchronized), (\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\", TimeNotSynchronized), (\"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\", TimeNotSynchronized), (\"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45\", TimeNotSynchronized), (\"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a\", TimeNotSynchronized), (\"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811\", TimeNotSynchronized), (\"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7\", TimeNotSynchronized), (\"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\", TimeNotSynchronized), (\"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402\", TimeNotSynchronized), (\"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344\", TimeNotSynchronized), (\"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d\", TimeNotSynchronized), (\"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e\", TimeNotSynchronized), (\"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b\", TimeNotSynchronized), (\"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f\", TimeNotSynchronized), (\"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301\", TimeNotSynchronized), (\"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd\", TimeNotSynchronized), (\"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b\", TimeNotSynchronized)]","latency_us":296314,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:46.304049513Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"1.00752718s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a05e08b7-edf5-4466-84a4-8a97a0031978\", \"content-length\": \"124\", \"date\": \"Wed, 15 Jan 2025 02:24:46 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a05e08b7-edf5-4466-84a4-8a97a0031978\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e94e881-2bab-4d68-8315-741d3cd1814a (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a4d0b75d-0c67-4224-af19-0510665c8775 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 44ac331a-da63-47e3-a636-214c3ca86b22 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 46c4b5d7-b839-4d27-ac2d-e1f6f6eae7d3, kind: Service { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 }, name: Name(\"ntp-fd5ec85c-1d4a-43a3-bd64-3bf287ecded3\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 9])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 77bd825f-cbb2-4b31-ba9f-d69e9e4cff59 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: a6c98ed9-62e0-43ae-9066-c1d4ee8e0596 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 788bf241-055b-45bf-8d67-e865c6324e46 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 82a65d0c-f391-4b1f-9c11-e3a397904f10 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 03e9536c-5029-447f-944a-4daff2909c7f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7468547b-b2f3-4993-afd8-72e9164a4301 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: b15378a9-5120-477b-abef-a170753103c1, kind: Service { id: 7468547b-b2f3-4993-afd8-72e9164a4301 }, name: Name(\"external-dns-7468547b-b2f3-4993-afd8-72e9164a4301\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 124])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beea2870-fa6b-4d58-b0eb-af5a33149d11, kind: Service { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a }, name: Name(\"external-dns-8b302e6b-b09c-4ef2-8768-1e5fae50d31a\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 41])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 357305ec-a785-4a6e-96cf-879dc3168076, kind: Service { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f }, name: Name(\"nexus-a1e38c68-2155-4ce9-b70a-66d434b3f24f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 213])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 56f43fd0-1c39-449d-9089-5f10b7acf59b, kind: Service { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d }, name: Name(\"nexus-b4bce764-527e-4a0d-9efa-46f5760fbb1d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 31])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 743f2600-b4a7-4963-8bef-692428b6800a (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 22195cd0-81ca-4c14-8294-e0bb8767fa51, kind: Service { id: 743f2600-b4a7-4963-8bef-692428b6800a }, name: Name(\"nexus-743f2600-b4a7-4963-8bef-692428b6800a\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 167, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f71d33aa-7320-48b9-81d2-899a100e78cd (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 40c9a9db-bd0a-45df-b2ea-f8a9ad278c76 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: eb0487cd-afd0-473b-b56c-c84ebdb9a402 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 683f3bcd-05eb-48ca-a731-4591f3875c45 (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 100f148b-d0a5-4957-b7f4-7b142438883b (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 1ecfbd35-39cb-454a-b9bf-b046feb4f811 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: 014247db-8886-4cf5-b68e-375be0dabc7f (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 7361bad6-7bc4-4252-9897-84b3278d100e (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6e9d55e6-6648-4adf-a2be-39839524cd72 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ceaca86-5562-442f-a7a7-bbfd684c302b (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 771664a2-5c9b-4063-8f72-848056c94344 (service), filesystem_pool: Some(ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37091b30-13af-44d8-ac11-483eada5eeca (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 25c723ea-c52b-44b6-8751-dbe925d1e7a7 (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External } } } }, OmicronZoneConfig { id: f12a4089-4d9a-4d0c-b3e6-8a12d533bb03 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:47.316334024Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:47.316613491Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:47.316623466Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:47.327988751Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:47.328036743Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:47.337763595Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:47.337804979Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:47.347548128Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:47.347587635Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:47.356652093Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:47.35668514Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:47.366825733Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:47.366863234Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:47.375760848Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:47.375806055Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:47.38479112Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:47.38483058Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:47.394581127Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:47.394619143Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:47.404125428Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:47.404160135Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:47.415157668Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:47.415195094Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:47.425758443Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:47.425802244Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:47.435330391Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:47.435367226Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:47.445755478Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:47.445796335Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:47.455400857Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:47.455439464Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:47.466094763Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:47.466123029Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:47.475573453Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:47.475677085Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:47.485170428Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:47.485212851Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:47.495335068Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:47.495368435Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:47.504870363Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:47.50490852Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:47.514361721Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:47.514402485Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:47.514622363Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:47.514831162Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:47.615632249Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:47.615693975Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:47.615717601Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:47.61572331Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:47.615729863Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:47.615735133Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:47.615745999Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:47.61575588Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:47.615759668Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:47.615765218Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:47.615781765Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:47.615786809Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:47.615797713Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:47.615801763Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:47.615805298Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:47.615810388Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:47.615814119Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:47.615826879Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:47.615830583Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:47.61583542Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"ecdfd50b-350f-4106-8fbd-d2591d8752eb","remote_addr":"[fd00:1122:3344:101::1]:53729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\", TimeNotSynchronized), (\"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b\", TimeNotSynchronized), (\"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344\", TimeNotSynchronized), (\"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca\", TimeNotSynchronized), (\"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7\", TimeNotSynchronized), (\"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\", TimeNotSynchronized), (\"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301\", TimeNotSynchronized), (\"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b\", TimeNotSynchronized), (\"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402\", TimeNotSynchronized), (\"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a\", TimeNotSynchronized), (\"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f\", TimeNotSynchronized), (\"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d\", TimeNotSynchronized), (\"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a\", TimeNotSynchronized), (\"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd\", TimeNotSynchronized), (\"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811\", TimeNotSynchronized), (\"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45\", TimeNotSynchronized), (\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\", TimeNotSynchronized), (\"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\", TimeNotSynchronized), (\"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e\", TimeNotSynchronized)]","latency_us":298635,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:47.616054749Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"896.767076ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ecdfd50b-350f-4106-8fbd-d2591d8752eb\", \"content-length\": \"124\", \"date\": \"Wed, 15 Jan 2025 02:24:47 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ecdfd50b-350f-4106-8fbd-d2591d8752eb\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e94e881-2bab-4d68-8315-741d3cd1814a (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a4d0b75d-0c67-4224-af19-0510665c8775 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 44ac331a-da63-47e3-a636-214c3ca86b22 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 46c4b5d7-b839-4d27-ac2d-e1f6f6eae7d3, kind: Service { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 }, name: Name(\"ntp-fd5ec85c-1d4a-43a3-bd64-3bf287ecded3\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 9])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 77bd825f-cbb2-4b31-ba9f-d69e9e4cff59 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: a6c98ed9-62e0-43ae-9066-c1d4ee8e0596 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 788bf241-055b-45bf-8d67-e865c6324e46 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 82a65d0c-f391-4b1f-9c11-e3a397904f10 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 03e9536c-5029-447f-944a-4daff2909c7f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7468547b-b2f3-4993-afd8-72e9164a4301 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: b15378a9-5120-477b-abef-a170753103c1, kind: Service { id: 7468547b-b2f3-4993-afd8-72e9164a4301 }, name: Name(\"external-dns-7468547b-b2f3-4993-afd8-72e9164a4301\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 124])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beea2870-fa6b-4d58-b0eb-af5a33149d11, kind: Service { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a }, name: Name(\"external-dns-8b302e6b-b09c-4ef2-8768-1e5fae50d31a\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 41])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 357305ec-a785-4a6e-96cf-879dc3168076, kind: Service { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f }, name: Name(\"nexus-a1e38c68-2155-4ce9-b70a-66d434b3f24f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 213])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 56f43fd0-1c39-449d-9089-5f10b7acf59b, kind: Service { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d }, name: Name(\"nexus-b4bce764-527e-4a0d-9efa-46f5760fbb1d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 31])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 743f2600-b4a7-4963-8bef-692428b6800a (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 22195cd0-81ca-4c14-8294-e0bb8767fa51, kind: Service { id: 743f2600-b4a7-4963-8bef-692428b6800a }, name: Name(\"nexus-743f2600-b4a7-4963-8bef-692428b6800a\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 167, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f71d33aa-7320-48b9-81d2-899a100e78cd (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 40c9a9db-bd0a-45df-b2ea-f8a9ad278c76 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: eb0487cd-afd0-473b-b56c-c84ebdb9a402 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 683f3bcd-05eb-48ca-a731-4591f3875c45 (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 100f148b-d0a5-4957-b7f4-7b142438883b (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 1ecfbd35-39cb-454a-b9bf-b046feb4f811 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: 014247db-8886-4cf5-b68e-375be0dabc7f (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 7361bad6-7bc4-4252-9897-84b3278d100e (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6e9d55e6-6648-4adf-a2be-39839524cd72 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ceaca86-5562-442f-a7a7-bbfd684c302b (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 771664a2-5c9b-4063-8f72-848056c94344 (service), filesystem_pool: Some(ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37091b30-13af-44d8-ac11-483eada5eeca (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 25c723ea-c52b-44b6-8751-dbe925d1e7a7 (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External } } } }, OmicronZoneConfig { id: f12a4089-4d9a-4d0c-b3e6-8a12d533bb03 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:48.516114231Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:48.516391795Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:48.516470693Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:48.528092825Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:48.530581368Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:48.539782964Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:48.539799136Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:48.549642765Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:48.549690606Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:48.558977599Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:48.559021043Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:48.56969568Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:48.569750796Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:48.579788599Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:48.5798312Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:48.589874621Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:48.589921075Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:48.602191509Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:48.602241216Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:48.613077832Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:48.613125017Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:48.622496299Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:48.622543081Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:48.633541694Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:48.633590314Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:48.643398795Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:48.643443759Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:48.654284284Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:48.654336973Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:48.664159718Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:48.664205779Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:48.675627022Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:48.678057437Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:48.685684229Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:48.68569968Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:48.69708255Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:48.697131864Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:48.707277111Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:48.707323257Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:48.716677282Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:48.716729802Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:48.727669188Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:48.727750387Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:48.727955846Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:48.728151071Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:48.828566958Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:48.828632893Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:48.828640506Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:48.828652816Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:48.828658488Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:48.828671567Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:48.828677117Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:48.828680998Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:48.828686304Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:48.828689895Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:48.828694882Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:48.82870002Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:48.828705411Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:48.828710689Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:48.828723214Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:48.828733162Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:48.828738439Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:48.828743278Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:48.828751575Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:48.828764569Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"f00b6af6-99d4-4889-8345-1c4d62faf028","remote_addr":"[fd00:1122:3344:101::1]:53729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811\", TimeNotSynchronized), (\"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca\", TimeNotSynchronized), (\"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e\", TimeNotSynchronized), (\"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45\", TimeNotSynchronized), (\"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a\", TimeNotSynchronized), (\"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\", TimeNotSynchronized), (\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\", TimeNotSynchronized), (\"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b\", TimeNotSynchronized), (\"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a\", TimeNotSynchronized), (\"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b\", TimeNotSynchronized), (\"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7\", TimeNotSynchronized), (\"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\", TimeNotSynchronized), (\"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301\", TimeNotSynchronized), (\"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402\", TimeNotSynchronized), (\"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f\", TimeNotSynchronized), (\"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d\", TimeNotSynchronized), (\"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\", TimeNotSynchronized), (\"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd\", TimeNotSynchronized), (\"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344\", TimeNotSynchronized)]","latency_us":311682,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:48.828930511Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"2.055289912s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f00b6af6-99d4-4889-8345-1c4d62faf028\", \"content-length\": \"124\", \"date\": \"Wed, 15 Jan 2025 02:24:48 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f00b6af6-99d4-4889-8345-1c4d62faf028\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e94e881-2bab-4d68-8315-741d3cd1814a (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a4d0b75d-0c67-4224-af19-0510665c8775 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 44ac331a-da63-47e3-a636-214c3ca86b22 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 46c4b5d7-b839-4d27-ac2d-e1f6f6eae7d3, kind: Service { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 }, name: Name(\"ntp-fd5ec85c-1d4a-43a3-bd64-3bf287ecded3\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 9])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 77bd825f-cbb2-4b31-ba9f-d69e9e4cff59 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: a6c98ed9-62e0-43ae-9066-c1d4ee8e0596 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 788bf241-055b-45bf-8d67-e865c6324e46 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 82a65d0c-f391-4b1f-9c11-e3a397904f10 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 03e9536c-5029-447f-944a-4daff2909c7f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7468547b-b2f3-4993-afd8-72e9164a4301 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: b15378a9-5120-477b-abef-a170753103c1, kind: Service { id: 7468547b-b2f3-4993-afd8-72e9164a4301 }, name: Name(\"external-dns-7468547b-b2f3-4993-afd8-72e9164a4301\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 124])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beea2870-fa6b-4d58-b0eb-af5a33149d11, kind: Service { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a }, name: Name(\"external-dns-8b302e6b-b09c-4ef2-8768-1e5fae50d31a\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 41])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 357305ec-a785-4a6e-96cf-879dc3168076, kind: Service { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f }, name: Name(\"nexus-a1e38c68-2155-4ce9-b70a-66d434b3f24f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 213])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 56f43fd0-1c39-449d-9089-5f10b7acf59b, kind: Service { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d }, name: Name(\"nexus-b4bce764-527e-4a0d-9efa-46f5760fbb1d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 31])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 743f2600-b4a7-4963-8bef-692428b6800a (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 22195cd0-81ca-4c14-8294-e0bb8767fa51, kind: Service { id: 743f2600-b4a7-4963-8bef-692428b6800a }, name: Name(\"nexus-743f2600-b4a7-4963-8bef-692428b6800a\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 167, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f71d33aa-7320-48b9-81d2-899a100e78cd (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 40c9a9db-bd0a-45df-b2ea-f8a9ad278c76 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: eb0487cd-afd0-473b-b56c-c84ebdb9a402 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 683f3bcd-05eb-48ca-a731-4591f3875c45 (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 100f148b-d0a5-4957-b7f4-7b142438883b (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 1ecfbd35-39cb-454a-b9bf-b046feb4f811 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: 014247db-8886-4cf5-b68e-375be0dabc7f (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 7361bad6-7bc4-4252-9897-84b3278d100e (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6e9d55e6-6648-4adf-a2be-39839524cd72 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ceaca86-5562-442f-a7a7-bbfd684c302b (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 771664a2-5c9b-4063-8f72-848056c94344 (service), filesystem_pool: Some(ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37091b30-13af-44d8-ac11-483eada5eeca (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 25c723ea-c52b-44b6-8751-dbe925d1e7a7 (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External } } } }, OmicronZoneConfig { id: f12a4089-4d9a-4d0c-b3e6-8a12d533bb03 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:50.890566494Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:50.890930927Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:50.89094096Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:50.901686564Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:50.901739525Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:50.912103999Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:50.912167573Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:50.922845657Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:50.922973904Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:50.93438544Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:50.934436835Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:50.94446455Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:50.944513639Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:50.954762478Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:50.954806626Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:50.964444651Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:50.964492118Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:50.973667157Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:50.973715468Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:50.984437443Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:50.984487019Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:50.99540865Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:50.995456444Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:51.006610584Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:51.006659035Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:51.019133186Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:51.019243955Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:51.030158115Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:51.030274574Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:51.04134447Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:51.041392414Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:51.052055642Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:51.052105801Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:51.062822843Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:51.062875409Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:51.073263823Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:51.073387231Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:51.085225043Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:51.085282064Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:51.095899008Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:51.09594745Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:51.10725752Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:51.107309439Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:51.107588018Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:51.10771268Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:51.203923897Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:51.204001337Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:51.204012549Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:51.204020293Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:51.20402874Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:51.204036569Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:51.204041209Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:51.204047013Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:51.204051719Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:51.204064114Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:51.204072157Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:51.204092651Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:51.204099158Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:51.204104043Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:51.204108496Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:51.204120243Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:51.204124986Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:51.204129411Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:51.204133884Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:51.204138197Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"b6d4f032-2143-4a49-9a55-a08b0cc78de5","remote_addr":"[fd00:1122:3344:101::1]:53729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d\", TimeNotSynchronized), (\"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b\", TimeNotSynchronized), (\"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e\", TimeNotSynchronized), (\"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b\", TimeNotSynchronized), (\"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca\", TimeNotSynchronized), (\"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7\", TimeNotSynchronized), (\"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\", TimeNotSynchronized), (\"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\", TimeNotSynchronized), (\"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402\", TimeNotSynchronized), (\"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f\", TimeNotSynchronized), (\"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45\", TimeNotSynchronized), (\"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301\", TimeNotSynchronized), (\"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a\", TimeNotSynchronized), (\"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\", TimeNotSynchronized), (\"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344\", TimeNotSynchronized), (\"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a\", TimeNotSynchronized), (\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\", TimeNotSynchronized), (\"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd\", TimeNotSynchronized), (\"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811\", TimeNotSynchronized)]","latency_us":312578,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:51.204279838Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"2.700845228s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b6d4f032-2143-4a49-9a55-a08b0cc78de5\", \"content-length\": \"124\", \"date\": \"Wed, 15 Jan 2025 02:24:50 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b6d4f032-2143-4a49-9a55-a08b0cc78de5\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:52.046629134Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"1.223718438s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e94e881-2bab-4d68-8315-741d3cd1814a (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a4d0b75d-0c67-4224-af19-0510665c8775 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 44ac331a-da63-47e3-a636-214c3ca86b22 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 46c4b5d7-b839-4d27-ac2d-e1f6f6eae7d3, kind: Service { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 }, name: Name(\"ntp-fd5ec85c-1d4a-43a3-bd64-3bf287ecded3\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 9])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 77bd825f-cbb2-4b31-ba9f-d69e9e4cff59 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: a6c98ed9-62e0-43ae-9066-c1d4ee8e0596 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 788bf241-055b-45bf-8d67-e865c6324e46 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 82a65d0c-f391-4b1f-9c11-e3a397904f10 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 03e9536c-5029-447f-944a-4daff2909c7f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7468547b-b2f3-4993-afd8-72e9164a4301 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: b15378a9-5120-477b-abef-a170753103c1, kind: Service { id: 7468547b-b2f3-4993-afd8-72e9164a4301 }, name: Name(\"external-dns-7468547b-b2f3-4993-afd8-72e9164a4301\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 124])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beea2870-fa6b-4d58-b0eb-af5a33149d11, kind: Service { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a }, name: Name(\"external-dns-8b302e6b-b09c-4ef2-8768-1e5fae50d31a\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 41])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 357305ec-a785-4a6e-96cf-879dc3168076, kind: Service { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f }, name: Name(\"nexus-a1e38c68-2155-4ce9-b70a-66d434b3f24f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 213])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 56f43fd0-1c39-449d-9089-5f10b7acf59b, kind: Service { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d }, name: Name(\"nexus-b4bce764-527e-4a0d-9efa-46f5760fbb1d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 31])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 743f2600-b4a7-4963-8bef-692428b6800a (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 22195cd0-81ca-4c14-8294-e0bb8767fa51, kind: Service { id: 743f2600-b4a7-4963-8bef-692428b6800a }, name: Name(\"nexus-743f2600-b4a7-4963-8bef-692428b6800a\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 167, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f71d33aa-7320-48b9-81d2-899a100e78cd (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 40c9a9db-bd0a-45df-b2ea-f8a9ad278c76 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: eb0487cd-afd0-473b-b56c-c84ebdb9a402 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 683f3bcd-05eb-48ca-a731-4591f3875c45 (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 100f148b-d0a5-4957-b7f4-7b142438883b (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 1ecfbd35-39cb-454a-b9bf-b046feb4f811 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: 014247db-8886-4cf5-b68e-375be0dabc7f (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 7361bad6-7bc4-4252-9897-84b3278d100e (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6e9d55e6-6648-4adf-a2be-39839524cd72 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ceaca86-5562-442f-a7a7-bbfd684c302b (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 771664a2-5c9b-4063-8f72-848056c94344 (service), filesystem_pool: Some(ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37091b30-13af-44d8-ac11-483eada5eeca (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 25c723ea-c52b-44b6-8751-dbe925d1e7a7 (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External } } } }, OmicronZoneConfig { id: f12a4089-4d9a-4d0c-b3e6-8a12d533bb03 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:53.913281654Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:53.913556059Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:53.91356325Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:53.925203869Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:53.925259351Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:53.936430096Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:53.93648599Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:53.947091583Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:53.947138891Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:53.95755926Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:53.957602752Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:53.968619081Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:53.968671986Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:53.979231779Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:53.979281234Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:53.989342534Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:53.989387047Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:53.999774694Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:53.999822592Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:54.010290577Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:54.01034454Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:54.021188602Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:54.021303784Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:54.031899474Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:54.031946453Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:54.042104447Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:54.042150386Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:54.053981331Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:54.054043761Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:54.064460884Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:54.064511164Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:54.074503841Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:54.074550229Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:54.085955789Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:54.086002252Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:54.096132329Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:54.096186153Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:54.106582345Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:54.106637359Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:54.117354327Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:54.117402243Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:54.12808475Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:54.12816248Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:54.128354533Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:54.128489209Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:54.228358427Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:54.22843957Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:54.228449883Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:54.228474624Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:54.228481308Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:54.228489418Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:54.228509144Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:54.228530342Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:54.228538592Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:54.228546673Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:54.228556076Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:54.228562254Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:54.228568836Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:54.228575014Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:54.228579693Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:54.228587371Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:54.228592283Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:54.228600046Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:54.228606543Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:54.228651863Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"6c8c68c5-57c9-4c11-9d58-f955bfdc1b28","remote_addr":"[fd00:1122:3344:101::1]:53729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd\", TimeNotSynchronized), (\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\", TimeNotSynchronized), (\"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402\", TimeNotSynchronized), (\"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\", TimeNotSynchronized), (\"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e\", TimeNotSynchronized), (\"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\", TimeNotSynchronized), (\"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b\", TimeNotSynchronized), (\"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a\", TimeNotSynchronized), (\"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b\", TimeNotSynchronized), (\"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\", TimeNotSynchronized), (\"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a\", TimeNotSynchronized), (\"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301\", TimeNotSynchronized), (\"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca\", TimeNotSynchronized), (\"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7\", TimeNotSynchronized), (\"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f\", TimeNotSynchronized), (\"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344\", TimeNotSynchronized), (\"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45\", TimeNotSynchronized), (\"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811\", TimeNotSynchronized), (\"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d\", TimeNotSynchronized)]","latency_us":314441,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:54.228877669Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"1.459260305s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6c8c68c5-57c9-4c11-9d58-f955bfdc1b28\", \"content-length\": \"124\", \"date\": \"Wed, 15 Jan 2025 02:24:53 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6c8c68c5-57c9-4c11-9d58-f955bfdc1b28\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e94e881-2bab-4d68-8315-741d3cd1814a (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a4d0b75d-0c67-4224-af19-0510665c8775 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 44ac331a-da63-47e3-a636-214c3ca86b22 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 46c4b5d7-b839-4d27-ac2d-e1f6f6eae7d3, kind: Service { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 }, name: Name(\"ntp-fd5ec85c-1d4a-43a3-bd64-3bf287ecded3\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 9])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 77bd825f-cbb2-4b31-ba9f-d69e9e4cff59 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: a6c98ed9-62e0-43ae-9066-c1d4ee8e0596 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 788bf241-055b-45bf-8d67-e865c6324e46 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 82a65d0c-f391-4b1f-9c11-e3a397904f10 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 03e9536c-5029-447f-944a-4daff2909c7f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7468547b-b2f3-4993-afd8-72e9164a4301 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: b15378a9-5120-477b-abef-a170753103c1, kind: Service { id: 7468547b-b2f3-4993-afd8-72e9164a4301 }, name: Name(\"external-dns-7468547b-b2f3-4993-afd8-72e9164a4301\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 124])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beea2870-fa6b-4d58-b0eb-af5a33149d11, kind: Service { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a }, name: Name(\"external-dns-8b302e6b-b09c-4ef2-8768-1e5fae50d31a\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 41])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 357305ec-a785-4a6e-96cf-879dc3168076, kind: Service { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f }, name: Name(\"nexus-a1e38c68-2155-4ce9-b70a-66d434b3f24f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 213])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 56f43fd0-1c39-449d-9089-5f10b7acf59b, kind: Service { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d }, name: Name(\"nexus-b4bce764-527e-4a0d-9efa-46f5760fbb1d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 31])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 743f2600-b4a7-4963-8bef-692428b6800a (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 22195cd0-81ca-4c14-8294-e0bb8767fa51, kind: Service { id: 743f2600-b4a7-4963-8bef-692428b6800a }, name: Name(\"nexus-743f2600-b4a7-4963-8bef-692428b6800a\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 167, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f71d33aa-7320-48b9-81d2-899a100e78cd (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 40c9a9db-bd0a-45df-b2ea-f8a9ad278c76 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: eb0487cd-afd0-473b-b56c-c84ebdb9a402 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 683f3bcd-05eb-48ca-a731-4591f3875c45 (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 100f148b-d0a5-4957-b7f4-7b142438883b (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 1ecfbd35-39cb-454a-b9bf-b046feb4f811 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: 014247db-8886-4cf5-b68e-375be0dabc7f (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 7361bad6-7bc4-4252-9897-84b3278d100e (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6e9d55e6-6648-4adf-a2be-39839524cd72 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ceaca86-5562-442f-a7a7-bbfd684c302b (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 771664a2-5c9b-4063-8f72-848056c94344 (service), filesystem_pool: Some(ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37091b30-13af-44d8-ac11-483eada5eeca (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 25c723ea-c52b-44b6-8751-dbe925d1e7a7 (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External } } } }, OmicronZoneConfig { id: f12a4089-4d9a-4d0c-b3e6-8a12d533bb03 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:55.693653583Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:55.694032953Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:55.694041888Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:55.705168824Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:55.705216272Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:55.714430389Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:55.714491413Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:55.723428476Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:55.723477565Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:55.733813678Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:55.733922922Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:55.744326528Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:55.74437888Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:55.754874586Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:55.754997275Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:55.764880328Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:55.76498952Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:55.774346909Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:55.774394479Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:55.784412045Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:55.784465924Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:55.794686441Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:55.794731488Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:55.804372814Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:55.804428222Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:55.814134263Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:55.814192474Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:55.823820673Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:55.823867258Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:55.83440182Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:55.834451255Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:55.843516604Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:55.843562899Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:55.854468181Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:55.85452027Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:55.864350332Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:55.864413691Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:55.874798512Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:55.87484971Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:55.885201737Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:55.885313355Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:55.894814524Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:55.894881819Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:55.895067197Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:55.895194531Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:55.992972358Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:55.993063711Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:55.993073733Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:55.993097226Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:55.99310618Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:55.993115902Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:55.993124237Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:55.993132375Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:55.993140334Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:55.993148603Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:55.993155288Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:55.993163501Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:55.99317611Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:55.993181585Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:55.993191457Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:55.993197046Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:55.993201096Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:55.993206149Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:55.993211371Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:55.993273548Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"eb04b206-fd48-4cbd-b972-a42ca272b445","remote_addr":"[fd00:1122:3344:101::1]:53729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e\", TimeNotSynchronized), (\"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a\", TimeNotSynchronized), (\"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301\", TimeNotSynchronized), (\"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402\", TimeNotSynchronized), (\"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f\", TimeNotSynchronized), (\"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\", TimeNotSynchronized), (\"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d\", TimeNotSynchronized), (\"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7\", TimeNotSynchronized), (\"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a\", TimeNotSynchronized), (\"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45\", TimeNotSynchronized), (\"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b\", TimeNotSynchronized), (\"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\", TimeNotSynchronized), (\"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811\", TimeNotSynchronized), (\"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca\", TimeNotSynchronized), (\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\", TimeNotSynchronized), (\"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344\", TimeNotSynchronized), (\"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd\", TimeNotSynchronized), (\"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\", TimeNotSynchronized), (\"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b\", TimeNotSynchronized)]","latency_us":298682,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:24:55.99343542Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"3.820025506s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"eb04b206-fd48-4cbd-b972-a42ca272b445\", \"content-length\": \"124\", \"date\": \"Wed, 15 Jan 2025 02:24:55 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"eb04b206-fd48-4cbd-b972-a42ca272b445\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e94e881-2bab-4d68-8315-741d3cd1814a (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a4d0b75d-0c67-4224-af19-0510665c8775 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 44ac331a-da63-47e3-a636-214c3ca86b22 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 46c4b5d7-b839-4d27-ac2d-e1f6f6eae7d3, kind: Service { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 }, name: Name(\"ntp-fd5ec85c-1d4a-43a3-bd64-3bf287ecded3\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 9])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 77bd825f-cbb2-4b31-ba9f-d69e9e4cff59 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: a6c98ed9-62e0-43ae-9066-c1d4ee8e0596 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 788bf241-055b-45bf-8d67-e865c6324e46 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 82a65d0c-f391-4b1f-9c11-e3a397904f10 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 03e9536c-5029-447f-944a-4daff2909c7f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7468547b-b2f3-4993-afd8-72e9164a4301 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: b15378a9-5120-477b-abef-a170753103c1, kind: Service { id: 7468547b-b2f3-4993-afd8-72e9164a4301 }, name: Name(\"external-dns-7468547b-b2f3-4993-afd8-72e9164a4301\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 124])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beea2870-fa6b-4d58-b0eb-af5a33149d11, kind: Service { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a }, name: Name(\"external-dns-8b302e6b-b09c-4ef2-8768-1e5fae50d31a\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 41])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 357305ec-a785-4a6e-96cf-879dc3168076, kind: Service { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f }, name: Name(\"nexus-a1e38c68-2155-4ce9-b70a-66d434b3f24f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 213])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 56f43fd0-1c39-449d-9089-5f10b7acf59b, kind: Service { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d }, name: Name(\"nexus-b4bce764-527e-4a0d-9efa-46f5760fbb1d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 31])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 743f2600-b4a7-4963-8bef-692428b6800a (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 22195cd0-81ca-4c14-8294-e0bb8767fa51, kind: Service { id: 743f2600-b4a7-4963-8bef-692428b6800a }, name: Name(\"nexus-743f2600-b4a7-4963-8bef-692428b6800a\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 167, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f71d33aa-7320-48b9-81d2-899a100e78cd (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 40c9a9db-bd0a-45df-b2ea-f8a9ad278c76 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: eb0487cd-afd0-473b-b56c-c84ebdb9a402 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 683f3bcd-05eb-48ca-a731-4591f3875c45 (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 100f148b-d0a5-4957-b7f4-7b142438883b (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 1ecfbd35-39cb-454a-b9bf-b046feb4f811 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: 014247db-8886-4cf5-b68e-375be0dabc7f (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 7361bad6-7bc4-4252-9897-84b3278d100e (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6e9d55e6-6648-4adf-a2be-39839524cd72 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ceaca86-5562-442f-a7a7-bbfd684c302b (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 771664a2-5c9b-4063-8f72-848056c94344 (service), filesystem_pool: Some(ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37091b30-13af-44d8-ac11-483eada5eeca (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 25c723ea-c52b-44b6-8751-dbe925d1e7a7 (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External } } } }, OmicronZoneConfig { id: f12a4089-4d9a-4d0c-b3e6-8a12d533bb03 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:59.824616798Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:59.824957027Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:59.824966308Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:59.836644325Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:59.836702658Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:59.847387695Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:59.847439127Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:59.858482112Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:59.858526625Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:59.868888417Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:59.868934807Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:59.87965802Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:59.879709292Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:59.890172022Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:59.89254575Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:59.900279059Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:59.900292474Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:59.909911129Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:59.909964888Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:59.919998628Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:59.920043619Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:59.929361703Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:59.929406095Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:59.938706444Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:59.938758673Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:59.94874339Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:59.948788354Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:59.958922446Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:59.95897206Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:59.968838493Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:59.968884291Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:59.978470968Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:59.978514572Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:59.98804765Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:59.988111477Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:24:59.999034548Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:00.002083351Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:00.010576832Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:00.010592639Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:00.021551076Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:00.021598617Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:00.034720363Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:00.034788679Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:00.034991757Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:00.035111948Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:00.137108109Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:00.137170426Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:00.137177242Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:00.137189673Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:00.137194324Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:00.137204684Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:00.137209202Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:00.137213627Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:00.137219487Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:00.137223612Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:00.137239756Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:00.137247041Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:00.137252956Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:00.13725846Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:00.13726401Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:00.137270882Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:00.137289848Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:00.137295107Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:00.137300621Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:00.137397448Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"61538929-8fb7-4243-ad7f-996a62a23c11","remote_addr":"[fd00:1122:3344:101::1]:53729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301\", TimeNotSynchronized), (\"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f\", TimeNotSynchronized), (\"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45\", TimeNotSynchronized), (\"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d\", TimeNotSynchronized), (\"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a\", TimeNotSynchronized), (\"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a\", TimeNotSynchronized), (\"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811\", TimeNotSynchronized), (\"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402\", TimeNotSynchronized), (\"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd\", TimeNotSynchronized), (\"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\", TimeNotSynchronized), (\"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e\", TimeNotSynchronized), (\"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\", TimeNotSynchronized), (\"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344\", TimeNotSynchronized), (\"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b\", TimeNotSynchronized), (\"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca\", TimeNotSynchronized), (\"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7\", TimeNotSynchronized), (\"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\", TimeNotSynchronized), (\"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b\", TimeNotSynchronized), (\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\", TimeNotSynchronized)]","latency_us":311821,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:00.137565527Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"3.192776098s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"61538929-8fb7-4243-ad7f-996a62a23c11\", \"content-length\": \"124\", \"date\": \"Wed, 15 Jan 2025 02:25:00 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"61538929-8fb7-4243-ad7f-996a62a23c11\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e94e881-2bab-4d68-8315-741d3cd1814a (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a4d0b75d-0c67-4224-af19-0510665c8775 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 44ac331a-da63-47e3-a636-214c3ca86b22 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 46c4b5d7-b839-4d27-ac2d-e1f6f6eae7d3, kind: Service { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 }, name: Name(\"ntp-fd5ec85c-1d4a-43a3-bd64-3bf287ecded3\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 9])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 77bd825f-cbb2-4b31-ba9f-d69e9e4cff59 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: a6c98ed9-62e0-43ae-9066-c1d4ee8e0596 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 788bf241-055b-45bf-8d67-e865c6324e46 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 82a65d0c-f391-4b1f-9c11-e3a397904f10 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 03e9536c-5029-447f-944a-4daff2909c7f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7468547b-b2f3-4993-afd8-72e9164a4301 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: b15378a9-5120-477b-abef-a170753103c1, kind: Service { id: 7468547b-b2f3-4993-afd8-72e9164a4301 }, name: Name(\"external-dns-7468547b-b2f3-4993-afd8-72e9164a4301\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 124])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beea2870-fa6b-4d58-b0eb-af5a33149d11, kind: Service { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a }, name: Name(\"external-dns-8b302e6b-b09c-4ef2-8768-1e5fae50d31a\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 41])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 357305ec-a785-4a6e-96cf-879dc3168076, kind: Service { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f }, name: Name(\"nexus-a1e38c68-2155-4ce9-b70a-66d434b3f24f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 213])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 56f43fd0-1c39-449d-9089-5f10b7acf59b, kind: Service { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d }, name: Name(\"nexus-b4bce764-527e-4a0d-9efa-46f5760fbb1d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 31])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 743f2600-b4a7-4963-8bef-692428b6800a (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 22195cd0-81ca-4c14-8294-e0bb8767fa51, kind: Service { id: 743f2600-b4a7-4963-8bef-692428b6800a }, name: Name(\"nexus-743f2600-b4a7-4963-8bef-692428b6800a\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 167, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f71d33aa-7320-48b9-81d2-899a100e78cd (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 40c9a9db-bd0a-45df-b2ea-f8a9ad278c76 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: eb0487cd-afd0-473b-b56c-c84ebdb9a402 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 683f3bcd-05eb-48ca-a731-4591f3875c45 (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 100f148b-d0a5-4957-b7f4-7b142438883b (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 1ecfbd35-39cb-454a-b9bf-b046feb4f811 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: 014247db-8886-4cf5-b68e-375be0dabc7f (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 7361bad6-7bc4-4252-9897-84b3278d100e (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6e9d55e6-6648-4adf-a2be-39839524cd72 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ceaca86-5562-442f-a7a7-bbfd684c302b (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 771664a2-5c9b-4063-8f72-848056c94344 (service), filesystem_pool: Some(ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37091b30-13af-44d8-ac11-483eada5eeca (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 25c723ea-c52b-44b6-8751-dbe925d1e7a7 (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External } } } }, OmicronZoneConfig { id: f12a4089-4d9a-4d0c-b3e6-8a12d533bb03 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:03.339911286Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:03.340204572Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:03.340218456Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:03.352544942Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:03.354923893Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:03.362630879Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:03.36264903Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:03.372190477Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:03.372243738Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:03.380763924Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:03.380812066Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:03.392030909Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:03.392092504Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:03.401164359Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:03.401212868Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:03.412019362Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:03.412065038Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:03.4211795Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:03.421228963Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:03.431976353Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:03.4320286Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:03.441295877Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:03.441350404Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:03.452191248Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:03.452240834Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:03.463058011Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:03.463108863Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:03.472625522Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:03.472673608Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:03.484455229Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:03.48451327Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:03.495011341Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:03.495065257Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:03.505102037Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:03.505151979Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:03.513929403Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:03.513978088Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:03.523918718Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:03.523965031Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:03.535623129Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:03.535668055Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:03.546503525Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:03.546574083Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:03.546769717Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:03.546973456Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:03.646908625Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:03.64695821Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:03.646964229Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:03.646968889Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:03.646973689Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:03.646977664Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:03.646982024Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:03.646986711Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:03.646995796Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:03.64700008Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:03.647004833Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:03.647009906Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:03.647013862Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:03.647029022Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:03.647036831Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:03.647040122Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:03.647046759Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:03.647051616Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:03.647054982Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:03.647090645Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"1de707f9-ae9e-4ba0-8634-b0124345de0c","remote_addr":"[fd00:1122:3344:101::1]:53729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301\", TimeNotSynchronized), (\"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a\", TimeNotSynchronized), (\"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402\", TimeNotSynchronized), (\"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d\", TimeNotSynchronized), (\"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca\", TimeNotSynchronized), (\"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811\", TimeNotSynchronized), (\"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b\", TimeNotSynchronized), (\"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45\", TimeNotSynchronized), (\"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b\", TimeNotSynchronized), (\"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344\", TimeNotSynchronized), (\"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7\", TimeNotSynchronized), (\"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f\", TimeNotSynchronized), (\"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\", TimeNotSynchronized), (\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\", TimeNotSynchronized), (\"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a\", TimeNotSynchronized), (\"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e\", TimeNotSynchronized), (\"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\", TimeNotSynchronized), (\"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\", TimeNotSynchronized), (\"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd\", TimeNotSynchronized)]","latency_us":306250,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:03.647297689Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"2.64940063s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1de707f9-ae9e-4ba0-8634-b0124345de0c\", \"content-length\": \"124\", \"date\": \"Wed, 15 Jan 2025 02:25:03 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1de707f9-ae9e-4ba0-8634-b0124345de0c\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e94e881-2bab-4d68-8315-741d3cd1814a (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a4d0b75d-0c67-4224-af19-0510665c8775 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 44ac331a-da63-47e3-a636-214c3ca86b22 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 46c4b5d7-b839-4d27-ac2d-e1f6f6eae7d3, kind: Service { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 }, name: Name(\"ntp-fd5ec85c-1d4a-43a3-bd64-3bf287ecded3\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 9])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 77bd825f-cbb2-4b31-ba9f-d69e9e4cff59 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: a6c98ed9-62e0-43ae-9066-c1d4ee8e0596 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 788bf241-055b-45bf-8d67-e865c6324e46 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 82a65d0c-f391-4b1f-9c11-e3a397904f10 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 03e9536c-5029-447f-944a-4daff2909c7f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7468547b-b2f3-4993-afd8-72e9164a4301 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: b15378a9-5120-477b-abef-a170753103c1, kind: Service { id: 7468547b-b2f3-4993-afd8-72e9164a4301 }, name: Name(\"external-dns-7468547b-b2f3-4993-afd8-72e9164a4301\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 124])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beea2870-fa6b-4d58-b0eb-af5a33149d11, kind: Service { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a }, name: Name(\"external-dns-8b302e6b-b09c-4ef2-8768-1e5fae50d31a\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 41])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 357305ec-a785-4a6e-96cf-879dc3168076, kind: Service { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f }, name: Name(\"nexus-a1e38c68-2155-4ce9-b70a-66d434b3f24f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 213])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 56f43fd0-1c39-449d-9089-5f10b7acf59b, kind: Service { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d }, name: Name(\"nexus-b4bce764-527e-4a0d-9efa-46f5760fbb1d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 31])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 743f2600-b4a7-4963-8bef-692428b6800a (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 22195cd0-81ca-4c14-8294-e0bb8767fa51, kind: Service { id: 743f2600-b4a7-4963-8bef-692428b6800a }, name: Name(\"nexus-743f2600-b4a7-4963-8bef-692428b6800a\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 167, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f71d33aa-7320-48b9-81d2-899a100e78cd (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 40c9a9db-bd0a-45df-b2ea-f8a9ad278c76 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: eb0487cd-afd0-473b-b56c-c84ebdb9a402 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 683f3bcd-05eb-48ca-a731-4591f3875c45 (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 100f148b-d0a5-4957-b7f4-7b142438883b (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 1ecfbd35-39cb-454a-b9bf-b046feb4f811 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: 014247db-8886-4cf5-b68e-375be0dabc7f (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 7361bad6-7bc4-4252-9897-84b3278d100e (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6e9d55e6-6648-4adf-a2be-39839524cd72 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ceaca86-5562-442f-a7a7-bbfd684c302b (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 771664a2-5c9b-4063-8f72-848056c94344 (service), filesystem_pool: Some(ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37091b30-13af-44d8-ac11-483eada5eeca (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 25c723ea-c52b-44b6-8751-dbe925d1e7a7 (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External } } } }, OmicronZoneConfig { id: f12a4089-4d9a-4d0c-b3e6-8a12d533bb03 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:06.304657273Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:06.304942947Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:06.304949725Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:06.31699938Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:06.317041221Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:06.32797261Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:06.328009746Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:06.33874356Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:06.338779486Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:06.349398167Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:06.349508923Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:06.359775896Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:06.359824573Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:06.370606043Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:06.370638341Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:06.381795208Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:06.381830449Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:06.384955643Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:36824"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:06.385214972Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"55ed2f8a-26fa-4aea-9ff3-c331041f1a9b","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:36824","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":156,"response_code":"200"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:06.392629713Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:06.392645772Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:06.402816186Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:06.402852806Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:06.413040648Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:06.413076968Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:06.422768704Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:06.422806186Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:06.433130923Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:06.433162555Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:06.444245161Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:06.444293415Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:06.454411714Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:06.454451372Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:06.464702063Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:06.464735195Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:06.475755921Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:06.475853021Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:06.485763787Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:06.485798795Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:06.496250402Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:06.496291034Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:06.506200922Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:06.506234617Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:06.516983503Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:06.517031148Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:06.517256689Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:06.517382016Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:06.621012193Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:06.621067638Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:06.621072879Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:06.621076179Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:06.621084036Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:06.621088311Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:06.621096758Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:06.62111699Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:06.62112238Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:06.62112644Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:06.621131024Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:06.621136077Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:06.621139218Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:06.621143371Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:06.621149756Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:06.621154003Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:06.621161363Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:06.621231038Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:06.6212352Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:06.621273095Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"87be2669-5000-4226-ae50-61ef3c04f5eb","remote_addr":"[fd00:1122:3344:101::1]:53729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f\", TimeNotSynchronized), (\"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b\", TimeNotSynchronized), (\"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a\", TimeNotSynchronized), (\"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402\", TimeNotSynchronized), (\"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\", TimeNotSynchronized), (\"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811\", TimeNotSynchronized), (\"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b\", TimeNotSynchronized), (\"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344\", TimeNotSynchronized), (\"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd\", TimeNotSynchronized), (\"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d\", TimeNotSynchronized), (\"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca\", TimeNotSynchronized), (\"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7\", TimeNotSynchronized), (\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\", TimeNotSynchronized), (\"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\", TimeNotSynchronized), (\"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\", TimeNotSynchronized), (\"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e\", TimeNotSynchronized), (\"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45\", TimeNotSynchronized), (\"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301\", TimeNotSynchronized), (\"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a\", TimeNotSynchronized)]","latency_us":315616,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:06.621411661Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"8.18528182s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"87be2669-5000-4226-ae50-61ef3c04f5eb\", \"content-length\": \"124\", \"date\": \"Wed, 15 Jan 2025 02:25:06 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"87be2669-5000-4226-ae50-61ef3c04f5eb\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:08.35019743Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"2.272324861s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e94e881-2bab-4d68-8315-741d3cd1814a (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a4d0b75d-0c67-4224-af19-0510665c8775 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 44ac331a-da63-47e3-a636-214c3ca86b22 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 46c4b5d7-b839-4d27-ac2d-e1f6f6eae7d3, kind: Service { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 }, name: Name(\"ntp-fd5ec85c-1d4a-43a3-bd64-3bf287ecded3\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 9])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 77bd825f-cbb2-4b31-ba9f-d69e9e4cff59 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: a6c98ed9-62e0-43ae-9066-c1d4ee8e0596 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 788bf241-055b-45bf-8d67-e865c6324e46 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 82a65d0c-f391-4b1f-9c11-e3a397904f10 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 03e9536c-5029-447f-944a-4daff2909c7f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7468547b-b2f3-4993-afd8-72e9164a4301 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: b15378a9-5120-477b-abef-a170753103c1, kind: Service { id: 7468547b-b2f3-4993-afd8-72e9164a4301 }, name: Name(\"external-dns-7468547b-b2f3-4993-afd8-72e9164a4301\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 124])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beea2870-fa6b-4d58-b0eb-af5a33149d11, kind: Service { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a }, name: Name(\"external-dns-8b302e6b-b09c-4ef2-8768-1e5fae50d31a\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 41])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 357305ec-a785-4a6e-96cf-879dc3168076, kind: Service { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f }, name: Name(\"nexus-a1e38c68-2155-4ce9-b70a-66d434b3f24f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 213])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 56f43fd0-1c39-449d-9089-5f10b7acf59b, kind: Service { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d }, name: Name(\"nexus-b4bce764-527e-4a0d-9efa-46f5760fbb1d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 31])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 743f2600-b4a7-4963-8bef-692428b6800a (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 22195cd0-81ca-4c14-8294-e0bb8767fa51, kind: Service { id: 743f2600-b4a7-4963-8bef-692428b6800a }, name: Name(\"nexus-743f2600-b4a7-4963-8bef-692428b6800a\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 167, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f71d33aa-7320-48b9-81d2-899a100e78cd (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 40c9a9db-bd0a-45df-b2ea-f8a9ad278c76 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: eb0487cd-afd0-473b-b56c-c84ebdb9a402 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 683f3bcd-05eb-48ca-a731-4591f3875c45 (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 100f148b-d0a5-4957-b7f4-7b142438883b (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 1ecfbd35-39cb-454a-b9bf-b046feb4f811 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: 014247db-8886-4cf5-b68e-375be0dabc7f (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 7361bad6-7bc4-4252-9897-84b3278d100e (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6e9d55e6-6648-4adf-a2be-39839524cd72 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ceaca86-5562-442f-a7a7-bbfd684c302b (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 771664a2-5c9b-4063-8f72-848056c94344 (service), filesystem_pool: Some(ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37091b30-13af-44d8-ac11-483eada5eeca (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 25c723ea-c52b-44b6-8751-dbe925d1e7a7 (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External } } } }, OmicronZoneConfig { id: f12a4089-4d9a-4d0c-b3e6-8a12d533bb03 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:14.829622351Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:14.829926832Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:14.829933535Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:14.841661695Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:14.84170363Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:14.852982801Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:14.853019383Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:14.863182913Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:14.863223948Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:14.872531681Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:14.872569351Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:14.882770276Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:14.882808818Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:14.891979936Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:14.89203044Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:14.902252988Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:14.902289308Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:14.911787099Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:14.911823906Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:14.921886463Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:14.921919464Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:14.931277644Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:14.931310073Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:14.941698172Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:14.941741814Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:14.951505534Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:14.951543822Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:14.962780005Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:14.9628179Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:14.972358699Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:14.972392441Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:14.981805676Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:14.981840017Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:14.992124701Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:14.992156624Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:15.001378915Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:15.001414962Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:15.01227819Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:15.012313376Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:15.022602677Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:15.022637356Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:15.032734313Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:15.032783186Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:15.033024334Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:15.033152485Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:15.134199934Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:15.134251103Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:15.134266029Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:15.134274204Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:15.134279332Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:15.134283692Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:15.134295561Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:15.134298993Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:15.134302893Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:15.134306005Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:15.134310205Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:15.134313308Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:15.134318447Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:15.1343262Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:15.134329631Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:15.134332865Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:15.134335894Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:15.134349262Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:15.134355516Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:15.134358901Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"5c86f229-5823-4d29-b079-27e73b1dedf1","remote_addr":"[fd00:1122:3344:101::1]:53729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811\", TimeNotSynchronized), (\"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402\", TimeNotSynchronized), (\"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\", TimeNotSynchronized), (\"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301\", TimeNotSynchronized), (\"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344\", TimeNotSynchronized), (\"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7\", TimeNotSynchronized), (\"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b\", TimeNotSynchronized), (\"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\", TimeNotSynchronized), (\"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd\", TimeNotSynchronized), (\"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a\", TimeNotSynchronized), (\"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d\", TimeNotSynchronized), (\"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca\", TimeNotSynchronized), (\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\", TimeNotSynchronized), (\"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\", TimeNotSynchronized), (\"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e\", TimeNotSynchronized), (\"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a\", TimeNotSynchronized), (\"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b\", TimeNotSynchronized), (\"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f\", TimeNotSynchronized), (\"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45\", TimeNotSynchronized)]","latency_us":303780,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:15.134496904Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"8.808251639s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5c86f229-5823-4d29-b079-27e73b1dedf1\", \"content-length\": \"124\", \"date\": \"Wed, 15 Jan 2025 02:25:14 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5c86f229-5823-4d29-b079-27e73b1dedf1\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e94e881-2bab-4d68-8315-741d3cd1814a (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a4d0b75d-0c67-4224-af19-0510665c8775 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 44ac331a-da63-47e3-a636-214c3ca86b22 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 46c4b5d7-b839-4d27-ac2d-e1f6f6eae7d3, kind: Service { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 }, name: Name(\"ntp-fd5ec85c-1d4a-43a3-bd64-3bf287ecded3\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 9])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 77bd825f-cbb2-4b31-ba9f-d69e9e4cff59 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: a6c98ed9-62e0-43ae-9066-c1d4ee8e0596 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 788bf241-055b-45bf-8d67-e865c6324e46 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 82a65d0c-f391-4b1f-9c11-e3a397904f10 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 03e9536c-5029-447f-944a-4daff2909c7f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7468547b-b2f3-4993-afd8-72e9164a4301 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: b15378a9-5120-477b-abef-a170753103c1, kind: Service { id: 7468547b-b2f3-4993-afd8-72e9164a4301 }, name: Name(\"external-dns-7468547b-b2f3-4993-afd8-72e9164a4301\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 124])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beea2870-fa6b-4d58-b0eb-af5a33149d11, kind: Service { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a }, name: Name(\"external-dns-8b302e6b-b09c-4ef2-8768-1e5fae50d31a\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 41])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 357305ec-a785-4a6e-96cf-879dc3168076, kind: Service { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f }, name: Name(\"nexus-a1e38c68-2155-4ce9-b70a-66d434b3f24f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 213])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 56f43fd0-1c39-449d-9089-5f10b7acf59b, kind: Service { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d }, name: Name(\"nexus-b4bce764-527e-4a0d-9efa-46f5760fbb1d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 31])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 743f2600-b4a7-4963-8bef-692428b6800a (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 22195cd0-81ca-4c14-8294-e0bb8767fa51, kind: Service { id: 743f2600-b4a7-4963-8bef-692428b6800a }, name: Name(\"nexus-743f2600-b4a7-4963-8bef-692428b6800a\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 167, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f71d33aa-7320-48b9-81d2-899a100e78cd (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 40c9a9db-bd0a-45df-b2ea-f8a9ad278c76 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: eb0487cd-afd0-473b-b56c-c84ebdb9a402 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 683f3bcd-05eb-48ca-a731-4591f3875c45 (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 100f148b-d0a5-4957-b7f4-7b142438883b (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 1ecfbd35-39cb-454a-b9bf-b046feb4f811 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: 014247db-8886-4cf5-b68e-375be0dabc7f (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 7361bad6-7bc4-4252-9897-84b3278d100e (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6e9d55e6-6648-4adf-a2be-39839524cd72 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ceaca86-5562-442f-a7a7-bbfd684c302b (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 771664a2-5c9b-4063-8f72-848056c94344 (service), filesystem_pool: Some(ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37091b30-13af-44d8-ac11-483eada5eeca (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 25c723ea-c52b-44b6-8751-dbe925d1e7a7 (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External } } } }, OmicronZoneConfig { id: f12a4089-4d9a-4d0c-b3e6-8a12d533bb03 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:23.967337912Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:23.967584331Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:23.967600587Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:23.980176175Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:23.980215092Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:23.990297175Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:23.990350802Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:24.000647238Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:24.000682648Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:24.009861642Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:24.009893603Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:24.018699394Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:24.018741517Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:24.027396538Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:24.027428545Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:24.037605257Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:24.037639458Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:24.046937368Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:24.046971391Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:24.056257338Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:24.056290789Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:24.065089666Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:24.065130748Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:24.073951688Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:24.073984961Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:24.083560736Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:24.083592265Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:24.092612715Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:24.092644516Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:24.101903313Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:24.101937354Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:24.110965373Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:24.110996771Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:24.120224869Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:24.120256425Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:24.129420478Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:24.129457821Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:24.138239513Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:24.138273995Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:24.147717622Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:24.147748776Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:24.15674631Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:24.156784083Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:24.156979792Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:24.157128033Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:24.255166132Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:24.255212249Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:24.255220396Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:24.255224156Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:24.255228703Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:24.25523624Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:24.25524781Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:24.255253538Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:24.255259097Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:24.255267282Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:24.255272644Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:24.255277876Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:24.25528278Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:24.25528772Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:24.255298539Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:24.25530407Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:24.255310155Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:24.255314814Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:24.255318358Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:24.255371741Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"7fdf1eab-047b-4e86-a9ef-fbafda84c4c1","remote_addr":"[fd00:1122:3344:101::1]:53729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d\", TimeNotSynchronized), (\"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca\", TimeNotSynchronized), (\"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b\", TimeNotSynchronized), (\"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\", TimeNotSynchronized), (\"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45\", TimeNotSynchronized), (\"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344\", TimeNotSynchronized), (\"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301\", TimeNotSynchronized), (\"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a\", TimeNotSynchronized), (\"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e\", TimeNotSynchronized), (\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\", TimeNotSynchronized), (\"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd\", TimeNotSynchronized), (\"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b\", TimeNotSynchronized), (\"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\", TimeNotSynchronized), (\"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7\", TimeNotSynchronized), (\"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811\", TimeNotSynchronized), (\"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\", TimeNotSynchronized), (\"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f\", TimeNotSynchronized), (\"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a\", TimeNotSynchronized), (\"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402\", TimeNotSynchronized)]","latency_us":287174,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:24.255491154Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"4.162818848s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7fdf1eab-047b-4e86-a9ef-fbafda84c4c1\", \"content-length\": \"124\", \"date\": \"Wed, 15 Jan 2025 02:25:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7fdf1eab-047b-4e86-a9ef-fbafda84c4c1\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:25.705983771Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"3.699137286s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e94e881-2bab-4d68-8315-741d3cd1814a (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a4d0b75d-0c67-4224-af19-0510665c8775 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 44ac331a-da63-47e3-a636-214c3ca86b22 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 46c4b5d7-b839-4d27-ac2d-e1f6f6eae7d3, kind: Service { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 }, name: Name(\"ntp-fd5ec85c-1d4a-43a3-bd64-3bf287ecded3\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 9])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 77bd825f-cbb2-4b31-ba9f-d69e9e4cff59 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: a6c98ed9-62e0-43ae-9066-c1d4ee8e0596 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 788bf241-055b-45bf-8d67-e865c6324e46 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 82a65d0c-f391-4b1f-9c11-e3a397904f10 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 03e9536c-5029-447f-944a-4daff2909c7f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7468547b-b2f3-4993-afd8-72e9164a4301 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: b15378a9-5120-477b-abef-a170753103c1, kind: Service { id: 7468547b-b2f3-4993-afd8-72e9164a4301 }, name: Name(\"external-dns-7468547b-b2f3-4993-afd8-72e9164a4301\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 124])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beea2870-fa6b-4d58-b0eb-af5a33149d11, kind: Service { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a }, name: Name(\"external-dns-8b302e6b-b09c-4ef2-8768-1e5fae50d31a\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 41])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 357305ec-a785-4a6e-96cf-879dc3168076, kind: Service { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f }, name: Name(\"nexus-a1e38c68-2155-4ce9-b70a-66d434b3f24f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 213])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 56f43fd0-1c39-449d-9089-5f10b7acf59b, kind: Service { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d }, name: Name(\"nexus-b4bce764-527e-4a0d-9efa-46f5760fbb1d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 31])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 743f2600-b4a7-4963-8bef-692428b6800a (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 22195cd0-81ca-4c14-8294-e0bb8767fa51, kind: Service { id: 743f2600-b4a7-4963-8bef-692428b6800a }, name: Name(\"nexus-743f2600-b4a7-4963-8bef-692428b6800a\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 167, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f71d33aa-7320-48b9-81d2-899a100e78cd (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 40c9a9db-bd0a-45df-b2ea-f8a9ad278c76 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: eb0487cd-afd0-473b-b56c-c84ebdb9a402 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 683f3bcd-05eb-48ca-a731-4591f3875c45 (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 100f148b-d0a5-4957-b7f4-7b142438883b (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 1ecfbd35-39cb-454a-b9bf-b046feb4f811 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: 014247db-8886-4cf5-b68e-375be0dabc7f (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 7361bad6-7bc4-4252-9897-84b3278d100e (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6e9d55e6-6648-4adf-a2be-39839524cd72 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ceaca86-5562-442f-a7a7-bbfd684c302b (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 771664a2-5c9b-4063-8f72-848056c94344 (service), filesystem_pool: Some(ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37091b30-13af-44d8-ac11-483eada5eeca (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 25c723ea-c52b-44b6-8751-dbe925d1e7a7 (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External } } } }, OmicronZoneConfig { id: f12a4089-4d9a-4d0c-b3e6-8a12d533bb03 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:28.43063276Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:28.430987098Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:28.430999763Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:28.443145385Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:28.443187518Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:28.453944485Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:28.454052539Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:28.463650906Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:28.463682716Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:28.47513119Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:28.475167753Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:28.485860721Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:28.48590455Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:28.49672683Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:28.496763694Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:28.506903889Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:28.50693719Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:28.516893872Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:28.516946908Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:28.527166245Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:28.527269487Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:28.538109819Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:28.538144582Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:28.548795292Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:28.548834771Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:28.559196687Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:28.559235332Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:28.5693541Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:28.569394067Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:28.579592698Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:28.579627734Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:28.589242473Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:28.589277752Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:28.601710664Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:28.601747546Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:28.61201114Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:28.612051762Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:28.622222386Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:28.622262371Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:28.632925928Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:28.632974811Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:28.642656639Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:28.6426946Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:28.642901043Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:28.643022808Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:28.743162363Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:28.743216636Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:28.743228533Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:28.743232977Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:28.74324118Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:28.743244836Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:28.743249666Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:28.743256125Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:28.743260147Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:28.743276769Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:28.743280069Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:28.743284081Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:28.743288263Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:28.743291301Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:28.74329971Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:28.743302982Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:28.743306873Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:28.743311091Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:28.743314298Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:28.743320476Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"cc2ed461-bc8b-4f38-ab1a-171703f504c2","remote_addr":"[fd00:1122:3344:101::1]:53729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7\", TimeNotSynchronized), (\"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a\", TimeNotSynchronized), (\"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b\", TimeNotSynchronized), (\"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e\", TimeNotSynchronized), (\"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\", TimeNotSynchronized), (\"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d\", TimeNotSynchronized), (\"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811\", TimeNotSynchronized), (\"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\", TimeNotSynchronized), (\"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402\", TimeNotSynchronized), (\"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f\", TimeNotSynchronized), (\"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\", TimeNotSynchronized), (\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\", TimeNotSynchronized), (\"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301\", TimeNotSynchronized), (\"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd\", TimeNotSynchronized), (\"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45\", TimeNotSynchronized), (\"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a\", TimeNotSynchronized), (\"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b\", TimeNotSynchronized), (\"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344\", TimeNotSynchronized), (\"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca\", TimeNotSynchronized)]","latency_us":311696,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:28.743446226Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"8.521869707s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"cc2ed461-bc8b-4f38-ab1a-171703f504c2\", \"content-length\": \"124\", \"date\": \"Wed, 15 Jan 2025 02:25:28 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"cc2ed461-bc8b-4f38-ab1a-171703f504c2\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:36.498540244Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:35168"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:36.499613972Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"3addda0e-ed5a-4109-bf42-8d89a0fdcf11","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:35168","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":27,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e94e881-2bab-4d68-8315-741d3cd1814a (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a4d0b75d-0c67-4224-af19-0510665c8775 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 44ac331a-da63-47e3-a636-214c3ca86b22 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 46c4b5d7-b839-4d27-ac2d-e1f6f6eae7d3, kind: Service { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 }, name: Name(\"ntp-fd5ec85c-1d4a-43a3-bd64-3bf287ecded3\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 9])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 77bd825f-cbb2-4b31-ba9f-d69e9e4cff59 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: a6c98ed9-62e0-43ae-9066-c1d4ee8e0596 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 788bf241-055b-45bf-8d67-e865c6324e46 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 82a65d0c-f391-4b1f-9c11-e3a397904f10 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 03e9536c-5029-447f-944a-4daff2909c7f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7468547b-b2f3-4993-afd8-72e9164a4301 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: b15378a9-5120-477b-abef-a170753103c1, kind: Service { id: 7468547b-b2f3-4993-afd8-72e9164a4301 }, name: Name(\"external-dns-7468547b-b2f3-4993-afd8-72e9164a4301\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 124])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beea2870-fa6b-4d58-b0eb-af5a33149d11, kind: Service { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a }, name: Name(\"external-dns-8b302e6b-b09c-4ef2-8768-1e5fae50d31a\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 41])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 357305ec-a785-4a6e-96cf-879dc3168076, kind: Service { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f }, name: Name(\"nexus-a1e38c68-2155-4ce9-b70a-66d434b3f24f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 213])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 56f43fd0-1c39-449d-9089-5f10b7acf59b, kind: Service { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d }, name: Name(\"nexus-b4bce764-527e-4a0d-9efa-46f5760fbb1d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 31])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 743f2600-b4a7-4963-8bef-692428b6800a (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 22195cd0-81ca-4c14-8294-e0bb8767fa51, kind: Service { id: 743f2600-b4a7-4963-8bef-692428b6800a }, name: Name(\"nexus-743f2600-b4a7-4963-8bef-692428b6800a\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 167, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f71d33aa-7320-48b9-81d2-899a100e78cd (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 40c9a9db-bd0a-45df-b2ea-f8a9ad278c76 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: eb0487cd-afd0-473b-b56c-c84ebdb9a402 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 683f3bcd-05eb-48ca-a731-4591f3875c45 (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 100f148b-d0a5-4957-b7f4-7b142438883b (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 1ecfbd35-39cb-454a-b9bf-b046feb4f811 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: 014247db-8886-4cf5-b68e-375be0dabc7f (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 7361bad6-7bc4-4252-9897-84b3278d100e (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6e9d55e6-6648-4adf-a2be-39839524cd72 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ceaca86-5562-442f-a7a7-bbfd684c302b (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 771664a2-5c9b-4063-8f72-848056c94344 (service), filesystem_pool: Some(ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37091b30-13af-44d8-ac11-483eada5eeca (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 25c723ea-c52b-44b6-8751-dbe925d1e7a7 (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External } } } }, OmicronZoneConfig { id: f12a4089-4d9a-4d0c-b3e6-8a12d533bb03 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:37.289603836Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:37.289903515Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:37.289910022Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:37.30189282Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:37.301933996Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:37.312698207Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:37.312736074Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:37.323801969Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:37.323842479Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:37.33379843Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:37.333834919Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:37.343686381Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:37.343720479Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:37.353230692Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:37.353267649Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:37.363618605Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:37.363652103Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:37.373404609Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:37.373449686Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:37.384469248Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:37.384505568Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:37.39457142Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:37.394604543Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:37.405062623Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:37.405163118Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:37.415205891Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:37.415238967Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:37.42498424Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:37.42502161Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:37.43543709Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:37.435473129Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:37.445146797Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:37.445189004Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:37.455038972Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:37.455074673Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:37.465202352Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:37.465240078Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:37.474974394Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:37.475008698Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:37.485723985Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:37.485760577Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:37.495405342Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:37.495440789Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:37.495644308Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:37.495763485Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:37.596397793Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:37.596446751Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:37.596453089Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:37.59646582Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:37.596476067Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:37.596481429Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:37.596486437Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:37.596490721Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:37.596495493Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:37.596499543Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:37.596504287Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:37.596512724Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:37.59651803Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:37.596523113Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:37.596528025Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:37.596532009Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:37.596536809Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:37.596541225Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:37.596552363Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:37.596671507Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"6513d064-67bf-42f4-8bf2-942b40c94d7c","remote_addr":"[fd00:1122:3344:101::1]:53729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd\", TimeNotSynchronized), (\"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\", TimeNotSynchronized), (\"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402\", TimeNotSynchronized), (\"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca\", TimeNotSynchronized), (\"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b\", TimeNotSynchronized), (\"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d\", TimeNotSynchronized), (\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\", TimeNotSynchronized), (\"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301\", TimeNotSynchronized), (\"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45\", TimeNotSynchronized), (\"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a\", TimeNotSynchronized), (\"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811\", TimeNotSynchronized), (\"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a\", TimeNotSynchronized), (\"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e\", TimeNotSynchronized), (\"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\", TimeNotSynchronized), (\"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b\", TimeNotSynchronized), (\"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344\", TimeNotSynchronized), (\"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7\", TimeNotSynchronized), (\"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f\", TimeNotSynchronized), (\"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\", TimeNotSynchronized)]","latency_us":306100,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:37.59679871Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"11.75466653s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6513d064-67bf-42f4-8bf2-942b40c94d7c\", \"content-length\": \"124\", \"date\": \"Wed, 15 Jan 2025 02:25:37 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6513d064-67bf-42f4-8bf2-942b40c94d7c\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:44.492785587Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"8.602378991s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e94e881-2bab-4d68-8315-741d3cd1814a (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a4d0b75d-0c67-4224-af19-0510665c8775 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 44ac331a-da63-47e3-a636-214c3ca86b22 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 46c4b5d7-b839-4d27-ac2d-e1f6f6eae7d3, kind: Service { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 }, name: Name(\"ntp-fd5ec85c-1d4a-43a3-bd64-3bf287ecded3\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 9])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 77bd825f-cbb2-4b31-ba9f-d69e9e4cff59 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: a6c98ed9-62e0-43ae-9066-c1d4ee8e0596 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 788bf241-055b-45bf-8d67-e865c6324e46 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 82a65d0c-f391-4b1f-9c11-e3a397904f10 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 03e9536c-5029-447f-944a-4daff2909c7f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7468547b-b2f3-4993-afd8-72e9164a4301 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: b15378a9-5120-477b-abef-a170753103c1, kind: Service { id: 7468547b-b2f3-4993-afd8-72e9164a4301 }, name: Name(\"external-dns-7468547b-b2f3-4993-afd8-72e9164a4301\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 124])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beea2870-fa6b-4d58-b0eb-af5a33149d11, kind: Service { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a }, name: Name(\"external-dns-8b302e6b-b09c-4ef2-8768-1e5fae50d31a\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 41])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 357305ec-a785-4a6e-96cf-879dc3168076, kind: Service { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f }, name: Name(\"nexus-a1e38c68-2155-4ce9-b70a-66d434b3f24f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 213])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 56f43fd0-1c39-449d-9089-5f10b7acf59b, kind: Service { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d }, name: Name(\"nexus-b4bce764-527e-4a0d-9efa-46f5760fbb1d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 31])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 743f2600-b4a7-4963-8bef-692428b6800a (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 22195cd0-81ca-4c14-8294-e0bb8767fa51, kind: Service { id: 743f2600-b4a7-4963-8bef-692428b6800a }, name: Name(\"nexus-743f2600-b4a7-4963-8bef-692428b6800a\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 167, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f71d33aa-7320-48b9-81d2-899a100e78cd (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 40c9a9db-bd0a-45df-b2ea-f8a9ad278c76 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: eb0487cd-afd0-473b-b56c-c84ebdb9a402 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 683f3bcd-05eb-48ca-a731-4591f3875c45 (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 100f148b-d0a5-4957-b7f4-7b142438883b (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 1ecfbd35-39cb-454a-b9bf-b046feb4f811 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: 014247db-8886-4cf5-b68e-375be0dabc7f (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 7361bad6-7bc4-4252-9897-84b3278d100e (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6e9d55e6-6648-4adf-a2be-39839524cd72 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ceaca86-5562-442f-a7a7-bbfd684c302b (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 771664a2-5c9b-4063-8f72-848056c94344 (service), filesystem_pool: Some(ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37091b30-13af-44d8-ac11-483eada5eeca (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 25c723ea-c52b-44b6-8751-dbe925d1e7a7 (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External } } } }, OmicronZoneConfig { id: f12a4089-4d9a-4d0c-b3e6-8a12d533bb03 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:49.384085685Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:49.384426021Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:49.384442549Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:49.399176684Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:49.399224807Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:49.411413648Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:49.411525432Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:49.424899528Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:49.424950913Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:49.435913816Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:49.435951842Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:49.447880162Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:49.447926335Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:49.460245834Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:49.460282126Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:49.47121818Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:49.471252859Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:49.481120491Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:49.481163889Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:49.494898825Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:49.49494032Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:49.508306072Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:49.508363682Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:49.523235703Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:49.523296905Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:49.537294294Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:49.537342585Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:49.551895103Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:49.551942485Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:49.564764672Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:49.564812008Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:49.578129147Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:49.578178957Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:49.590646069Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:49.590692326Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:49.602758538Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:49.602825946Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:49.616781845Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:49.616831094Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:49.632964714Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:49.633012312Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:49.646548865Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:49.646599913Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:49.646937225Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:49.647172793Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:49.77540276Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:49.775522707Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:49.775538991Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:49.775546745Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:49.775552839Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:49.775557855Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:49.775564511Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:49.775571017Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:49.775576081Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:49.775581021Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:49.775584949Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:49.775593293Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:49.775605143Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:49.775615334Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:49.775620744Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:49.77562561Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:49.775632838Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:49.775641922Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:49.77564706Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:49.775651242Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"0ff2590d-bcf8-4ae1-9486-5350a83318d2","remote_addr":"[fd00:1122:3344:101::1]:53729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b\", TimeNotSynchronized), (\"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a\", TimeNotSynchronized), (\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\", TimeNotSynchronized), (\"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45\", TimeNotSynchronized), (\"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344\", TimeNotSynchronized), (\"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7\", TimeNotSynchronized), (\"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811\", TimeNotSynchronized), (\"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\", TimeNotSynchronized), (\"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301\", TimeNotSynchronized), (\"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\", TimeNotSynchronized), (\"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402\", TimeNotSynchronized), (\"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\", TimeNotSynchronized), (\"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f\", TimeNotSynchronized), (\"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd\", TimeNotSynchronized), (\"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a\", TimeNotSynchronized), (\"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b\", TimeNotSynchronized), (\"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca\", TimeNotSynchronized), (\"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e\", TimeNotSynchronized), (\"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d\", TimeNotSynchronized)]","latency_us":390359,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:49.775780835Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"8.019496536s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0ff2590d-bcf8-4ae1-9486-5350a83318d2\", \"content-length\": \"124\", \"date\": \"Wed, 15 Jan 2025 02:25:49 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0ff2590d-bcf8-4ae1-9486-5350a83318d2\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e94e881-2bab-4d68-8315-741d3cd1814a (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a4d0b75d-0c67-4224-af19-0510665c8775 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 44ac331a-da63-47e3-a636-214c3ca86b22 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 46c4b5d7-b839-4d27-ac2d-e1f6f6eae7d3, kind: Service { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 }, name: Name(\"ntp-fd5ec85c-1d4a-43a3-bd64-3bf287ecded3\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 9])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 77bd825f-cbb2-4b31-ba9f-d69e9e4cff59 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: a6c98ed9-62e0-43ae-9066-c1d4ee8e0596 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 788bf241-055b-45bf-8d67-e865c6324e46 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 82a65d0c-f391-4b1f-9c11-e3a397904f10 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 03e9536c-5029-447f-944a-4daff2909c7f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7468547b-b2f3-4993-afd8-72e9164a4301 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: b15378a9-5120-477b-abef-a170753103c1, kind: Service { id: 7468547b-b2f3-4993-afd8-72e9164a4301 }, name: Name(\"external-dns-7468547b-b2f3-4993-afd8-72e9164a4301\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 124])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beea2870-fa6b-4d58-b0eb-af5a33149d11, kind: Service { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a }, name: Name(\"external-dns-8b302e6b-b09c-4ef2-8768-1e5fae50d31a\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 41])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 357305ec-a785-4a6e-96cf-879dc3168076, kind: Service { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f }, name: Name(\"nexus-a1e38c68-2155-4ce9-b70a-66d434b3f24f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 213])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 56f43fd0-1c39-449d-9089-5f10b7acf59b, kind: Service { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d }, name: Name(\"nexus-b4bce764-527e-4a0d-9efa-46f5760fbb1d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 31])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 743f2600-b4a7-4963-8bef-692428b6800a (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 22195cd0-81ca-4c14-8294-e0bb8767fa51, kind: Service { id: 743f2600-b4a7-4963-8bef-692428b6800a }, name: Name(\"nexus-743f2600-b4a7-4963-8bef-692428b6800a\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 167, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f71d33aa-7320-48b9-81d2-899a100e78cd (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 40c9a9db-bd0a-45df-b2ea-f8a9ad278c76 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: eb0487cd-afd0-473b-b56c-c84ebdb9a402 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 683f3bcd-05eb-48ca-a731-4591f3875c45 (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 100f148b-d0a5-4957-b7f4-7b142438883b (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 1ecfbd35-39cb-454a-b9bf-b046feb4f811 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: 014247db-8886-4cf5-b68e-375be0dabc7f (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 7361bad6-7bc4-4252-9897-84b3278d100e (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6e9d55e6-6648-4adf-a2be-39839524cd72 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ceaca86-5562-442f-a7a7-bbfd684c302b (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 771664a2-5c9b-4063-8f72-848056c94344 (service), filesystem_pool: Some(ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37091b30-13af-44d8-ac11-483eada5eeca (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 25c723ea-c52b-44b6-8751-dbe925d1e7a7 (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External } } } }, OmicronZoneConfig { id: f12a4089-4d9a-4d0c-b3e6-8a12d533bb03 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:57.817890203Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:57.818229724Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:57.818237506Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:57.830234924Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:57.830272331Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:57.840658372Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:57.840716902Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:57.851053258Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:57.851086175Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:57.860407772Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:57.860455484Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:57.869277452Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:57.869314194Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:57.879451787Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:57.879485538Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:57.888595527Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:57.888624553Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:57.898113791Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:57.898150345Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:57.907782737Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:57.907823979Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:57.917504444Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:57.917544477Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:57.926637895Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:57.926676717Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:57.935899369Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:57.935933204Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:57.945211894Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:57.945243583Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:57.954589266Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:57.954632635Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:57.963572877Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:57.963611382Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:57.972417519Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:57.972444107Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:57.981258827Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:57.981296169Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:57.990179543Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:57.990215404Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:57.999398661Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:57.999433068Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:58.008957302Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:58.008994831Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:58.009201396Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:58.009353229Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:58.104772361Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:58.104824807Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:58.104840716Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:58.104912213Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:58.104920923Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:58.104935342Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:58.104942495Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:58.104948524Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:58.104954205Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:58.104966599Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:58.104986213Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:58.104992466Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:58.104997772Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:58.105003669Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:58.105007588Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:58.105011713Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:58.105016597Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:58.105021454Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:58.105025251Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3329","zone":"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:25:58.105032367Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"420e3d86-994c-44ae-ab72-9c2f61b7fefd","remote_addr":"[fd00:1122:3344:101::1]:53729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca\", TimeNotSynchronized), (\"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd\", TimeNotSynchronized), (\"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a\", TimeNotSynchronized), (\"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45\", TimeNotSynchronized), (\"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7\", TimeNotSynchronized), (\"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d\", TimeNotSynchronized), (\"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\", TimeNotSynchronized), (\"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a\", TimeNotSynchronized), (\"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301\", TimeNotSynchronized), (\"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b\", TimeNotSynchronized), (\"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402\", TimeNotSynchronized), (\"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b\", TimeNotSynchronized), (\"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\", TimeNotSynchronized), (\"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f\", TimeNotSynchronized), (\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\", TimeNotSynchronized), (\"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811\", TimeNotSynchronized), (\"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e\", TimeNotSynchronized), (\"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\", TimeNotSynchronized), (\"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344\", TimeNotSynchronized)]","latency_us":286227,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:25:58.105163967Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:578","retry_after":"24.489991989s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"420e3d86-994c-44ae-ab72-9c2f61b7fefd\", \"content-length\": \"124\", \"date\": \"Wed, 15 Jan 2025 02:25:57 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"420e3d86-994c-44ae-ab72-9c2f61b7fefd\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:06.613444275Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:44346"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:06.613543783Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"1ab870e4-03dd-4e12-9ba1-90305b67f074","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:44346","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":37,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:08.194719122Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"9.142668627s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 0e94e881-2bab-4d68-8315-741d3cd1814a (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: a4d0b75d-0c67-4224-af19-0510665c8775 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 44ac331a-da63-47e3-a636-214c3ca86b22 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 46c4b5d7-b839-4d27-ac2d-e1f6f6eae7d3, kind: Service { id: fd5ec85c-1d4a-43a3-bd64-3bf287ecded3 }, name: Name(\"ntp-fd5ec85c-1d4a-43a3-bd64-3bf287ecded3\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 9])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 77bd825f-cbb2-4b31-ba9f-d69e9e4cff59 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: a6c98ed9-62e0-43ae-9066-c1d4ee8e0596 (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 788bf241-055b-45bf-8d67-e865c6324e46 (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 82a65d0c-f391-4b1f-9c11-e3a397904f10 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 03e9536c-5029-447f-944a-4daff2909c7f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7468547b-b2f3-4993-afd8-72e9164a4301 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: b15378a9-5120-477b-abef-a170753103c1, kind: Service { id: 7468547b-b2f3-4993-afd8-72e9164a4301 }, name: Name(\"external-dns-7468547b-b2f3-4993-afd8-72e9164a4301\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 124])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beea2870-fa6b-4d58-b0eb-af5a33149d11, kind: Service { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a }, name: Name(\"external-dns-8b302e6b-b09c-4ef2-8768-1e5fae50d31a\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 41])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 357305ec-a785-4a6e-96cf-879dc3168076, kind: Service { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f }, name: Name(\"nexus-a1e38c68-2155-4ce9-b70a-66d434b3f24f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 213])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 56f43fd0-1c39-449d-9089-5f10b7acf59b, kind: Service { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d }, name: Name(\"nexus-b4bce764-527e-4a0d-9efa-46f5760fbb1d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 31])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 743f2600-b4a7-4963-8bef-692428b6800a (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 22195cd0-81ca-4c14-8294-e0bb8767fa51, kind: Service { id: 743f2600-b4a7-4963-8bef-692428b6800a }, name: Name(\"nexus-743f2600-b4a7-4963-8bef-692428b6800a\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 167, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: f71d33aa-7320-48b9-81d2-899a100e78cd (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 40c9a9db-bd0a-45df-b2ea-f8a9ad278c76 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: eb0487cd-afd0-473b-b56c-c84ebdb9a402 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 683f3bcd-05eb-48ca-a731-4591f3875c45 (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 100f148b-d0a5-4957-b7f4-7b142438883b (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 1ecfbd35-39cb-454a-b9bf-b046feb4f811 (service), filesystem_pool: Some(ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External } } } }, OmicronZoneConfig { id: 014247db-8886-4cf5-b68e-375be0dabc7f (service), filesystem_pool: Some(ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External } } } }, OmicronZoneConfig { id: 7361bad6-7bc4-4252-9897-84b3278d100e (service), filesystem_pool: Some(ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External } } } }, OmicronZoneConfig { id: 6e9d55e6-6648-4adf-a2be-39839524cd72 (service), filesystem_pool: Some(ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ceaca86-5562-442f-a7a7-bbfd684c302b (service), filesystem_pool: Some(ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External } } } }, OmicronZoneConfig { id: 771664a2-5c9b-4063-8f72-848056c94344 (service), filesystem_pool: Some(ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External } } } }, OmicronZoneConfig { id: 37091b30-13af-44d8-ac11-483eada5eeca (service), filesystem_pool: Some(ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External } } } }, OmicronZoneConfig { id: 25c723ea-c52b-44b6-8751-dbe925d1e7a7 (service), filesystem_pool: Some(ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External } } } }, OmicronZoneConfig { id: f12a4089-4d9a-4d0c-b3e6-8a12d533bb03 (service), filesystem_pool: Some(ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:22.662763884Z","hostname":"sock","pid":1566,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:541"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:22.663218021Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:22.663239125Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:22.674961878Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:22.675003185Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:22.685580196Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:22.68562289Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:22.696708061Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:22.69674276Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:22.705934891Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:22.705967479Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:22.715020114Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:22.715056059Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:22.723617732Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:22.723654642Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:22.732346312Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:22.732377138Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Cockroach }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:22.743335339Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:22.743377322Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:22.753059396Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:22.753095425Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:22.762528012Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:22.762560178Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:22.772448947Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:22.772484938Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:22.781809206Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:22.781843782Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:22.790531823Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:22.790567186Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:22.800550153Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:22.80058231Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:22.809574682Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:22.809609295Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:22.817643206Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:22.817675045Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:22.826263117Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:22.826294543Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:22.835832383Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:22.835863724Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:22.844572218Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:22.84461404Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:1478","request":"NewFilesystemRequest { dataset_id: None, dataset_name: DatasetName { pool_name: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:22.853214086Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:22.853249441Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:22.853462587Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:22.853656143Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:22.967728765Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:22.968912193Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:22.968945053Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:22.970284788Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.009004595Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.009268508Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.009299624Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.009309637Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.009319144Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.009329137Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.177560326Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.177694364Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.177715561Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.177729005Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.212663218Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.212896221Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.212903852Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.236305141Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.23634327Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Configuring new Omicron zone: oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.236445817Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.241221839Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.247221967Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.251994185Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.257302672Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.262556336Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.267523996Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.410772947Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.41584016Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.42114176Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.425908889Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.43086904Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.435965185Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.440884039Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Configuring new Omicron zone: oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.446207764Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.450824306Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.456075259Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.462332499Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.468893085Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.475410426Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.482898791Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.550489471Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.601391684Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Installing Omicron zone: oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.601719538Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.609972177Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.61580187Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.622065172Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.629262475Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.63637333Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.644060007Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.704796453Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.764408592Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.766609223Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.766634911Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.82048933Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:447","port":"Port { inner: PortInner(PortData { name: \"opte1\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 214, 213]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.820514569Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:1264","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.821982668Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.822003077Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.877349699Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:447","port":"Port { inner: PortInner(PortData { name: \"opte2\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 167, 161]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.8773782Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:1264","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.879857268Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.879869174Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.935621982Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:447","port":"Port { inner: PortInner(PortData { name: \"opte3\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 238, 124]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.935645027Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:1264","zone_type":"external_dns"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.994041095Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.996531332Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:23.996548948Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:24.052014727Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:447","port":"Port { inner: PortInner(PortData { name: \"opte4\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 169, 41]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:24.052040462Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:1264","zone_type":"external_dns"} {"msg":"no nat entry found for: 10.151.2.166","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:24.052268843Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.166","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:24.052286938Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.2.168","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:24.052373086Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.168","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:24.052386981Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.2.164","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:24.052579979Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.164","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:24.052591924Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:24.054091281Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:24.05410525Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:24.110507794Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:447","port":"Port { inner: PortInner(PortData { name: \"opte5\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 200, 31]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:24.110535124Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:1264","zone_type":"nexus"} {"msg":"creation of nat entry successful for: 10.151.2.166","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:24.110774708Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"creation of nat entry successful for: 10.151.2.168","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:24.112355535Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"creation of nat entry successful for: 10.151.2.164","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:24.114527787Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:24.114761653Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:24.114773962Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:24.117096194Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"no nat entry found for: 10.151.2.165","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:24.369070945Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.165","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:24.369143182Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"Configuring new Omicron zone: oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:24.369670413Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:24.382420168Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:24.402315588Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"no nat entry found for: 10.151.2.167","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:24.413018299Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.167","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:24.413068756Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.2.165","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:24.4132126Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:24.417451543Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Installing Omicron zone: oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:24.474088494Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:24.483694744Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:24.493297293Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"creation of nat entry successful for: 10.151.2.167","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:24.50408273Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Configuring new Omicron zone: oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:24.508333624Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:24.519270483Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Installing Omicron zone: oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:24.569291881Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Configuring new Omicron zone: oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:24.595935704Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:24.649564795Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Profile for oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:37.931570963Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:40.251585166Z","hostname":"sock","pid":1566,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers)\"","delay":"44.142254213s"} {"msg":"Booting oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:40.39512837Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:40.687870639Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:40.688819255Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:40.690428478Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:40.690763787Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:40.69084722Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:40.697454868Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:40.70346209Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:40.708183037Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:41.118242413Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:41.1185149Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:41.294397195Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:41.294852448Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:41.306842971Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:41.307053687Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:41.307764046Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:41.308150457Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:41.360121019Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:41.37275547Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:41.372821592Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:41.373070189Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:41.381978839Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:41.388418073Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:41.394711142Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:41.400248022Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:41.53507334Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:41.535404719Z","hostname":"sock","pid":1566,"zone":"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:41.605373457Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:41.605573363Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301\") failed: Property not found. retry in 29.073777ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:42.229969591Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b\") failed: Property not found. retry in 28.168187ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:42.242472249Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7\") failed: Property not found. retry in 62.649789ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:42.252335705Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45\") failed: Property not found. retry in 53.994409ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:42.262510389Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\") failed: Property not found. retry in 50.411095ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:42.272415987Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811\") failed: Property not found. retry in 38.359194ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:42.2840261Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e\") failed: Property not found. retry in 37.543534ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:42.29438491Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a\") failed: Property not found. retry in 44.87444ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:42.307137514Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b\") failed: Property not found. retry in 92.222083ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:42.468546169Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b\") failed: Property not found. retry in 32.346914ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:42.547176671Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca\") failed: Property not found. retry in 51.554042ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:42.563138822Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402\") failed: Property not found. retry in 36.490807ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:42.582453683Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301\") failed: Property not found. retry in 92.250091ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:42.597580641Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\") failed: Property not found. retry in 30.74553ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:42.609134893Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd\") failed: Property not found. retry in 41.769038ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:42.619926974Z","hostname":"sock","pid":1566,"zone":"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\") failed: Property not found. retry in 37.370415ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:42.629824701Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344\") failed: Property not found. retry in 44.012868ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:42.642531114Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b\") failed: Property not found. retry in 272.683678ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:42.653895021Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7\") failed: Property not found. retry in 65.903007ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:42.663035247Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b\") failed: Property not found. retry in 132.296041ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:42.67240629Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca\") failed: Property not found. retry in 91.970684ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:42.686035448Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45\") failed: Property not found. retry in 122.380974ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:42.697402625Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402\") failed: Property not found. retry in 75.098574ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:42.709522242Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301\") failed: Property not found. retry in 122.420399ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:42.721789983Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\") failed: Property not found. retry in 119.125539ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:42.732000848Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\") failed: Property not found. retry in 62.367453ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:42.743325244Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd\") failed: Property not found. retry in 58.147493ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:42.754621439Z","hostname":"sock","pid":1566,"zone":"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811\") failed: Property not found. retry in 105.86512ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:42.765469551Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e\") failed: Property not found. retry in 119.805498ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:42.777114816Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a\") failed: Property not found. retry in 63.701394ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:42.786731875Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\") failed: Property not found. retry in 75.164079ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:42.797002242Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344\") failed: Property not found. retry in 72.105299ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:42.80914809Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7\") failed: Property not found. retry in 256.747217ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:42.819549441Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b\") failed: Property not found. retry in 292.750466ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:42.831794921Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca\") failed: Property not found. retry in 181.468523ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:42.841452392Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45\") failed: Property not found. retry in 187.923888ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:42.852008877Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402\") failed: Property not found. retry in 246.800274ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:42.862955322Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301\") failed: Property not found. retry in 235.363714ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:42.872835759Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\") failed: Property not found. retry in 221.965076ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:42.886506248Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\") failed: Property not found. retry in 120.764385ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:42.897295409Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd\") failed: Property not found. retry in 208.801798ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:42.906624371Z","hostname":"sock","pid":1566,"zone":"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811\") failed: Property not found. retry in 143.955238ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:42.919256272Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e\") failed: Property not found. retry in 260.499729ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:42.929395085Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a\") failed: Property not found. retry in 295.284657ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:42.940652339Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\") failed: Property not found. retry in 295.3423ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:42.950099674Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344\") failed: Property not found. retry in 268.138506ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:42.959553729Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b\") failed: Property not found. retry in 456.258697ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:42.96811237Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\") failed: Property not found. retry in 482.860278ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:43.028207501Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca\") failed: Property not found. retry in 356.958183ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:43.040278557Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45\") failed: Property not found. retry in 377.593837ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:43.051892811Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811\") failed: Property not found. retry in 418.10435ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:43.076120034Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7\") failed: Property not found. retry in 463.423529ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:43.085828646Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301\") failed: Property not found. retry in 956.835765ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:43.121092078Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\") failed: Property not found. retry in 447.002009ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:43.131596711Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b\") failed: Property not found. retry in 583.633466ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:43.146173325Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402\") failed: Property not found. retry in 551.605555ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:43.158018344Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd\") failed: Property not found. retry in 598.128775ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:43.168674412Z","hostname":"sock","pid":1566,"zone":"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e\") failed: Property not found. retry in 554.234973ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:43.201105727Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344\") failed: Property not found. retry in 283.519001ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:43.239042Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a\") failed: Property not found. retry in 483.892036ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:43.250829839Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\") failed: Property not found. retry in 227.553881ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:43.261522187Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca\") failed: Property not found. retry in 641.597313ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:43.419940642Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"Profile for oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:43.42024983Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:43.420693992Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:43.420743664Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:43.428231026Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:43.428360109Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b\") failed: Property not found. retry in 948.719737ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:43.450018874Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"Booting oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:43.450201768Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45\") failed: Property not found. retry in 531.008339ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:43.467748222Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811\") failed: Property not found. retry in 703.687999ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:43.581749148Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\") failed: Property not found. retry in 758.269543ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:43.596847546Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7\") failed: Property not found. retry in 723.811447ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:43.610432212Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\") failed: Property not found. retry in 632.829952ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:43.62480925Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\") failed: Property not found. retry in 961.804019ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:43.638685475Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344\") failed: Property not found. retry in 404.099447ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:43.653130704Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f\") failed: Property not found. retry in 66.667107ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:43.699364002Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402\") failed: Property not found. retry in 746.439318ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:43.723054561Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b\") failed: Property not found. retry in 842.224581ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:43.74229683Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d\") failed: Property not found. retry in 54.045606ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:43.753508683Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a\") failed: Property not found. retry in 570.747295ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:43.766545455Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f\") failed: Property not found. retry in 114.151779ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:43.776606207Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a\") failed: Property not found. retry in 64.182696ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:43.787007947Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e\") failed: Property not found. retry in 1.008804158s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:43.801789986Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd\") failed: Property not found. retry in 512.837192ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:43.813854621Z","hostname":"sock","pid":1566,"zone":"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d\") failed: Property not found. retry in 71.875747ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:43.823260635Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a\") failed: Property not found. retry in 140.318713ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:43.901334432Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f\") failed: Property not found. retry in 234.579992ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:43.913177142Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d\") failed: Property not found. retry in 274.477743ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:43.924317622Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45\") failed: Property not found. retry in 602.721732ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:44.016054308Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a\") failed: Property not found. retry in 144.92863ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:44.064446734Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca\") failed: Property not found. retry in 553.444057ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:44.077722102Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344\") failed: Property not found. retry in 1.037808854s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:44.093520898Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301\") failed: Property not found. retry in 594.012175ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:44.1095169Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f\") failed: Property not found. retry in 543.229165ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:44.166335463Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d\") failed: Property not found. retry in 485.717613ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:44.214285456Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a\") failed: Property not found. retry in 442.679527ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:44.228074129Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\") failed: Property not found. retry in 1.283900507s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:44.277437581Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811\") failed: Property not found. retry in 969.266055ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:44.302943629Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd\") failed: Property not found. retry in 1.308658s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:44.3452219Z","hostname":"sock","pid":1566,"zone":"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7\") failed: Property not found. retry in 1.302012894s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:44.362564358Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a\") failed: Property not found. retry in 1.001211177s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:44.38078961Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\") failed: Property not found. retry in 1.128438137s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:44.396085842Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b\") failed: Property not found. retry in 562.454071ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:44.413333738Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402\") failed: Property not found. retry in 1.377458212s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:44.488323951Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b\") failed: Property not found. retry in 616.698237ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:44.602389439Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\") failed: Property not found. retry in 508.992741ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:44.616971563Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45\") failed: Property not found. retry in 1.453159811s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:44.637088095Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca\") failed: Property not found. retry in 762.580885ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:44.650101687Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a\") failed: Property not found. retry in 1.096762825s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:44.682909132Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d\") failed: Property not found. retry in 668.394243ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:44.715574004Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f\") failed: Property not found. retry in 658.21526ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:44.730705922Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301\") failed: Property not found. retry in 1.416569879s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:44.744587606Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e\") failed: Property not found. retry in 901.431389ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:44.824562105Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b\") failed: Property not found. retry in 1.175774947s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:44.989603778Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\") failed: Property not found. retry in 1.297735951s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:45.138843995Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344\") failed: Property not found. retry in 718.279536ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:45.151376904Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b\") failed: Property not found. retry in 883.945615ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:45.231515206Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811\") failed: Property not found. retry in 596.631316ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:45.285303597Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a\") failed: Property not found. retry in 1.296160483s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:45.397955747Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f\") failed: Property not found. retry in 1.178118088s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:45.410788573Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d\") failed: Property not found. retry in 1.153750548s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:45.422063517Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca\") failed: Property not found. retry in 1.257405518s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:45.432386286Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\") failed: Property not found. retry in 579.542638ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:45.537266696Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\") failed: Property not found. retry in 1.129914975s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:45.572318602Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd\") failed: Property not found. retry in 1.359404555s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:45.669199426Z","hostname":"sock","pid":1566,"zone":"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7\") failed: Property not found. retry in 1.451189248s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:45.684111409Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e\") failed: Property not found. retry in 910.114167ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:45.738216519Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a\") failed: Property not found. retry in 1.325932614s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:45.791575699Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402\") failed: Property not found. retry in 572.10976ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:45.878790862Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344\") failed: Property not found. retry in 1.133803156s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:45.889682186Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811\") failed: Property not found. retry in 1.093102732s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:45.900083947Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45\") failed: Property not found. retry in 667.85291ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:46.104105602Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b\") failed: Property not found. retry in 655.673012ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:46.130012371Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\") failed: Property not found. retry in 664.386487ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:46.142547819Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301\") failed: Property not found. retry in 1.300932411s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:46.175019775Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b\") failed: Property not found. retry in 718.318622ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:46.186041333Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\") failed: Property not found. retry in 958.251518ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:46.448383972Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402\") failed: Property not found. retry in 1.338251747s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:46.463434938Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d\") failed: Property not found. retry in 878.969269ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:46.589406217Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f\") failed: Property not found. retry in 1.365352497s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:46.602207583Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e\") failed: Property not found. retry in 957.371786ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:46.663203909Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca\") failed: Property not found. retry in 1.077745874s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:46.702446118Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a\") failed: Property not found. retry in 1.480245567s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:46.715979243Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\") failed: Property not found. retry in 797.470845ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:46.729324703Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45\") failed: Property not found. retry in 785.654633ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:46.785746155Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b\") failed: Property not found. retry in 1.424178085s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:46.799759594Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\") failed: Property not found. retry in 696.388876ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:46.822302352Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b\") failed: Property not found. retry in 939.035586ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:46.918740264Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811\") failed: Property not found. retry in 1.020410886s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:47.007460048Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344\") failed: Property not found. retry in 932.919072ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:47.037838428Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd\") failed: Property not found. retry in 1.321359849s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:47.050059378Z","hostname":"sock","pid":1566,"zone":"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a\") failed: Property not found. retry in 593.760603ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:47.132982445Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7\") failed: Property not found. retry in 1.149458658s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:47.151033423Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\") failed: Property not found. retry in 1.161866125s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:47.42442512Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d\") failed: Property not found. retry in 1.144290877s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:47.483642198Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301\") failed: Property not found. retry in 930.252231ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:47.494182502Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\") failed: Property not found. retry in 1.326117171s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:47.534357776Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\") failed: Property not found. retry in 793.378636ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:47.551121288Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45\") failed: Property not found. retry in 1.198636846s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:47.588310112Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e\") failed: Property not found. retry in 1.112292619s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:47.635549437Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a\") failed: Property not found. retry in 784.765251ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:47.742307478Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca\") failed: Property not found. retry in 737.707835ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:47.794954328Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402\") failed: Property not found. retry in 980.61085ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:47.817141887Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b\") failed: Property not found. retry in 1.019887427s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:47.873429676Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f\") failed: Property not found. retry in 1.465688709s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:47.982682394Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344\") failed: Property not found. retry in 1.484006572s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:47.997077312Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811\") failed: Property not found. retry in 796.662853ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:48.043478845Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a\") failed: Property not found. retry in 677.199889ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:48.211834157Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b\") failed: Property not found. retry in 621.002272ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:48.239629697Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7\") failed: Property not found. retry in 1.032735237s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:48.315754451Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\") failed: Property not found. retry in 880.990727ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:48.358474854Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd\") failed: Property not found. retry in 1.045437348s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:48.382302937Z","hostname":"sock","pid":1566,"zone":"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301\") failed: Property not found. retry in 1.307035903s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:48.435817051Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a\") failed: Property not found. retry in 1.350816832s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:48.54991818Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca\") failed: Property not found. retry in 907.273389ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:48.568615235Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\") failed: Property not found. retry in 596.135919ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:48.634614626Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d\") failed: Property not found. retry in 1.457138203s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:48.771020046Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e\") failed: Property not found. retry in 1.464066721s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:48.815315022Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45\") failed: Property not found. retry in 608.101509ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:48.824350846Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402\") failed: Property not found. retry in 666.800432ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:48.834598652Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811\") failed: Property not found. retry in 1.164003353s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:48.853716658Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b\") failed: Property not found. retry in 849.649589ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:48.890022518Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\") failed: Property not found. retry in 1.40291568s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:48.899430112Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b\") failed: Property not found. retry in 758.538506ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:48.911042245Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a\") failed: Property not found. retry in 694.265531ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:48.922779862Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"Profile for oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:48.947499949Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:26:48.947624812Z","hostname":"sock","pid":1566,"zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\") failed: Property not found. retry in 31.087519ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:49.195441199Z","hostname":"sock","pid":1566,"zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\") failed: Property not found. retry in 135.941756ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:49.237239827Z","hostname":"sock","pid":1566,"zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\") failed: Property not found. retry in 569.43692ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:49.250220917Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\") failed: Property not found. retry in 640.231505ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:49.261935623Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7\") failed: Property not found. retry in 1.447621245s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:49.35964062Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\") failed: Property not found. retry in 104.513689ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:49.385739644Z","hostname":"sock","pid":1566,"zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd\") failed: Property not found. retry in 1.297743181s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:49.455951269Z","hostname":"sock","pid":1566,"zone":"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f\") failed: Property not found. retry in 1.406778128s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:49.470214953Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45\") failed: Property not found. retry in 1.488629528s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:49.482896896Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca\") failed: Property not found. retry in 951.588285ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:49.494067427Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344\") failed: Property not found. retry in 1.470556533s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:49.50530221Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\") failed: Property not found. retry in 323.471556ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:49.514689704Z","hostname":"sock","pid":1566,"zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402\") failed: Property not found. retry in 704.49912ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:49.523413729Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a\") failed: Property not found. retry in 921.708648ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:49.628756362Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b\") failed: Property not found. retry in 869.028849ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:49.681925497Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b\") failed: Property not found. retry in 1.30443132s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:49.754096704Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301\") failed: Property not found. retry in 1.159003008s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:49.766032596Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\") failed: Property not found. retry in 1.1103312s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:49.832737626Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\") failed: Property not found. retry in 702.775106ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:49.854294158Z","hostname":"sock","pid":1566,"zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a\") failed: Property not found. retry in 661.519064ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:49.916849616Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\") failed: Property not found. retry in 1.102324894s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:49.929327053Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811\") failed: Property not found. retry in 591.730386ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:50.031996383Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d\") failed: Property not found. retry in 1.280090717s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:50.241097395Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402\") failed: Property not found. retry in 878.068023ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:50.253470089Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e\") failed: Property not found. retry in 1.473089149s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:50.294550608Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\") failed: Property not found. retry in 1.184573759s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:50.315343079Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca\") failed: Property not found. retry in 664.91612ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:50.459877679Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b\") failed: Property not found. retry in 907.583139ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:50.56220636Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a\") failed: Property not found. retry in 717.47786ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:50.575299984Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\") failed: Property not found. retry in 1.279547455s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:50.590056402Z","hostname":"sock","pid":1566,"zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a\") failed: Property not found. retry in 1.395739422s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:50.604211854Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811\") failed: Property not found. retry in 941.019458ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:50.636593307Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd\") failed: Property not found. retry in 1.081789602s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:50.767223652Z","hostname":"sock","pid":1566,"zone":"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7\") failed: Property not found. retry in 591.676294ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:50.825371482Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f\") failed: Property not found. retry in 627.729126ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:50.893662054Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301\") failed: Property not found. retry in 1.101501898s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:50.937320042Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\") failed: Property not found. retry in 1.110290773s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:50.958233367Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45\") failed: Property not found. retry in 900.851355ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:50.986173711Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344\") failed: Property not found. retry in 570.880701ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:50.997545658Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\") failed: Property not found. retry in 523.646247ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:51.047913727Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b\") failed: Property not found. retry in 1.075441755s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:51.069576032Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca\") failed: Property not found. retry in 1.279149488s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:51.143783936Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402\") failed: Property not found. retry in 865.538069ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:51.155230795Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a\") failed: Property not found. retry in 1.156379876s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:51.311947085Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7\") failed: Property not found. retry in 965.174173ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:51.432414877Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b\") failed: Property not found. retry in 596.151167ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:51.486623251Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\") failed: Property not found. retry in 728.504771ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:51.512000586Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f\") failed: Property not found. retry in 545.80169ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:51.533132197Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d\") failed: Property not found. retry in 501.119748ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:51.546761854Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344\") failed: Property not found. retry in 531.638039ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:51.58031141Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\") failed: Property not found. retry in 709.915633ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:51.590729892Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811\") failed: Property not found. retry in 1.328759381s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:51.601456041Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e\") failed: Property not found. retry in 765.710814ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:51.778734197Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd\") failed: Property not found. retry in 1.402210893s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:51.85995447Z","hostname":"sock","pid":1566,"zone":"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\") failed: Property not found. retry in 803.226234ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:51.879995681Z","hostname":"sock","pid":1566,"zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45\") failed: Property not found. retry in 1.116851375s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:51.896371123Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a\") failed: Property not found. retry in 698.253591ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:52.008515781Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402\") failed: Property not found. retry in 900.467013ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:52.027511333Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301\") failed: Property not found. retry in 873.546036ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:52.047156064Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d\") failed: Property not found. retry in 583.790133ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:52.057800991Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\") failed: Property not found. retry in 519.125454ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:52.077919484Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f\") failed: Property not found. retry in 1.191095968s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:52.087295658Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b\") failed: Property not found. retry in 1.237730955s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:52.096063824Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344\") failed: Property not found. retry in 1.083886618s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:52.121521362Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b\") failed: Property not found. retry in 1.026200352s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:52.155348225Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\") failed: Property not found. retry in 817.5865ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:52.249238319Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\") failed: Property not found. retry in 707.117574ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:52.310282617Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7\") failed: Property not found. retry in 877.994053ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:52.405914539Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca\") failed: Property not found. retry in 832.07888ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:52.431405177Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a\") failed: Property not found. retry in 816.06944ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:52.479368652Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e\") failed: Property not found. retry in 1.212153418s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:52.554804379Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\") failed: Property not found. retry in 708.268045ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:52.606508094Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d\") failed: Property not found. retry in 1.333215721s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:52.651569201Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\") failed: Property not found. retry in 1.192447446s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:52.691613592Z","hostname":"sock","pid":1566,"zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a\") failed: Property not found. retry in 1.087158441s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:52.7167231Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301\") failed: Property not found. retry in 702.37111ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:52.931389655Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402\") failed: Property not found. retry in 1.145991254s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:52.940195603Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811\") failed: Property not found. retry in 1.159218992s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:52.948391194Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45\") failed: Property not found. retry in 1.043490327s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:53.021386885Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\") failed: Property not found. retry in 1.276486924s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:53.030192353Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\") failed: Property not found. retry in 965.424323ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:53.07749785Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b\") failed: Property not found. retry in 889.063698ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:53.191132097Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344\") failed: Property not found. retry in 1.193838777s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:53.212817913Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd\") failed: Property not found. retry in 1.175070303s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:53.270952052Z","hostname":"sock","pid":1566,"zone":"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca\") failed: Property not found. retry in 1.147892409s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:53.279914884Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f\") failed: Property not found. retry in 1.1433675s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:53.287805287Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7\") failed: Property not found. retry in 1.314789148s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:53.295063283Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a\") failed: Property not found. retry in 1.441857336s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:53.303154702Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\") failed: Property not found. retry in 1.050917823s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:53.322198586Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b\") failed: Property not found. retry in 1.074586895s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:53.341004333Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301\") failed: Property not found. retry in 685.45523ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:53.644059655Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e\") failed: Property not found. retry in 1.110405251s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:53.781962048Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a\") failed: Property not found. retry in 926.105133ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:53.81204958Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\") failed: Property not found. retry in 794.504044ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:53.892269896Z","hostname":"sock","pid":1566,"zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d\") failed: Property not found. retry in 667.221609ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:53.995162924Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\") failed: Property not found. retry in 627.894198ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:54.052905923Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45\") failed: Property not found. retry in 601.015227ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:54.073862929Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b\") failed: Property not found. retry in 1.031104878s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:54.089615194Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402\") failed: Property not found. retry in 1.060661549s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:54.097935279Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811\") failed: Property not found. retry in 544.770743ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:54.116080948Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\") failed: Property not found. retry in 1.434929043s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:54.315627385Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301\") failed: Property not found. retry in 1.090493513s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:54.338188914Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\") failed: Property not found. retry in 1.293651245s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:54.382505671Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344\") failed: Property not found. retry in 733.387213ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:54.414679859Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b\") failed: Property not found. retry in 1.228509391s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:54.42469503Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca\") failed: Property not found. retry in 1.147331687s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:54.436263862Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f\") failed: Property not found. retry in 1.065809836s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:54.444504605Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd\") failed: Property not found. retry in 525.644966ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:54.455353937Z","hostname":"sock","pid":1566,"zone":"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7\") failed: Property not found. retry in 779.20066ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:54.618827151Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811\") failed: Property not found. retry in 1.432825725s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:54.671094862Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d\") failed: Property not found. retry in 545.0209ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:54.679758286Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45\") failed: Property not found. retry in 1.216025867s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:54.690018613Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\") failed: Property not found. retry in 715.28027ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:54.704048362Z","hostname":"sock","pid":1566,"zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\") failed: Property not found. retry in 522.598141ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:54.715158291Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a\") failed: Property not found. retry in 934.916582ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:54.749043596Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a\") failed: Property not found. retry in 1.197910135s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:54.757901945Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e\") failed: Property not found. retry in 851.780016ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:54.901622546Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd\") failed: Property not found. retry in 746.927344ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:54.990858005Z","hostname":"sock","pid":1566,"zone":"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b\") failed: Property not found. retry in 1.152539579s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:55.131413619Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344\") failed: Property not found. retry in 937.493665ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:55.157273998Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402\") failed: Property not found. retry in 981.299633ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:55.168072799Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d\") failed: Property not found. retry in 1.303501271s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:55.23480801Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\") failed: Property not found. retry in 641.852597ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:55.24740575Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7\") failed: Property not found. retry in 1.119126767s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:55.407219105Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\") failed: Property not found. retry in 946.925374ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:55.428572231Z","hostname":"sock","pid":1566,"zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301\") failed: Property not found. retry in 519.396743ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:55.438031387Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f\") failed: Property not found. retry in 754.653175ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:55.518724955Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca\") failed: Property not found. retry in 1.406227161s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:55.592181168Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\") failed: Property not found. retry in 586.727816ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:55.684283455Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a\") failed: Property not found. retry in 657.509692ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:55.712358123Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd\") failed: Property not found. retry in 938.394291ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:55.746437753Z","hostname":"sock","pid":1566,"zone":"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\") failed: Property not found. retry in 765.441367ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:55.759702011Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e\") failed: Property not found. retry in 820.170622ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:55.76780625Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72\") failed: Property not found. retry in 593.317862ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:55.899730021Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45\") failed: Property not found. retry in 633.117244ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:55.914975073Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a\") failed: Property not found. retry in 644.70655ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:55.964995573Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301\") failed: Property not found. retry in 1.092651876s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:55.97600457Z","hostname":"sock","pid":1566,"zone":"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344\") failed: Property not found. retry in 1.369166641s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:56.104013386Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811\") failed: Property not found. retry in 950.125393ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:56.113633016Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402\") failed: Property not found. retry in 781.45353ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:56.158710272Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f\") failed: Property not found. retry in 1.464192377s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:56.280383675Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f\") failed: Property not found. retry in 817.192357ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:56.29168693Z","hostname":"sock","pid":1566,"zone":"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\") failed: Property not found. retry in 1.071141511s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:56.395539944Z","hostname":"sock","pid":1566,"zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03\") failed: Property not found. retry in 1.3084101s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:56.535667536Z","hostname":"sock","pid":1566,"zone":"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\") failed: Property not found. retry in 1.139326707s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:57.478936265Z","hostname":"sock","pid":1566,"zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\") failed: Property not found. retry in 1.039495519s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:58.630471961Z","hostname":"sock","pid":1566,"zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\") failed: Property not found. retry in 826.14211ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:26:59.682631032Z","hostname":"sock","pid":1566,"zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\") failed: Property not found. retry in 1.39795423s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:27:00.519574717Z","hostname":"sock","pid":1566,"zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\") failed: Property not found. retry in 1.294100229s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:27:01.935357362Z","hostname":"sock","pid":1566,"zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ce66c06a-7d54-47a6-860d-59da68080fee)","v":0,"name":"SledAgent","level":50,"time":"2025-01-15T02:27:02.917660958Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ce66c06a-7d54-47a6-860d-59da68080fee)","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:27:02.920538726Z","hostname":"sock","pid":1566,"component":"NexusNotifierTask","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/nexus.rs:527"} {"msg":"wait for service Some(\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\") failed: Property not found. retry in 930.945109ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:27:03.240482619Z","hostname":"sock","pid":1566,"zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ce66c06a-7d54-47a6-860d-59da68080fee)","v":0,"name":"SledAgent","level":50,"time":"2025-01-15T02:27:03.937245522Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"wait for service Some(\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\") failed: Property not found. retry in 1.004913404s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:27:04.182328045Z","hostname":"sock","pid":1566,"zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ce66c06a-7d54-47a6-860d-59da68080fee)","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:27:04.187782792Z","hostname":"sock","pid":1566,"component":"NexusNotifierTask","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/nexus.rs:527"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ce66c06a-7d54-47a6-860d-59da68080fee)","v":0,"name":"SledAgent","level":50,"time":"2025-01-15T02:27:04.940811213Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"wait for service Some(\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\") failed: Property not found. retry in 1.391081974s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:27:05.199296438Z","hostname":"sock","pid":1566,"zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ce66c06a-7d54-47a6-860d-59da68080fee)","v":0,"name":"SledAgent","level":50,"time":"2025-01-15T02:27:05.944169939Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/ce66c06a-7d54-47a6-860d-59da68080fee)","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:27:06.187216517Z","hostname":"sock","pid":1566,"component":"NexusNotifierTask","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/nexus.rs:527"} {"msg":"wait for service Some(\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\") failed: Property not found. retry in 1.249047547s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:27:06.602207646Z","hostname":"sock","pid":1566,"zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/ce66c06a-7d54-47a6-860d-59da68080fee)","v":0,"name":"SledAgent","level":50,"time":"2025-01-15T02:27:06.951308567Z","hostname":"sock","pid":1566,"component":"ProbeManager","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"wait for service Some(\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\") failed: Property not found. retry in 894.774618ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:27:07.862029943Z","hostname":"sock","pid":1566,"zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\") failed: Property not found. retry in 1.328583154s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:27:08.769304816Z","hostname":"sock","pid":1566,"zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:08.778295038Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:62620"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:08.782586144Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"b1f2b8c8-d9d4-44a4-946a-24b042f9fd54","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:62620","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":1076,"response_code":"200"} {"msg":"wait for service Some(\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\") failed: Property not found. retry in 544.058272ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:27:10.10877271Z","hostname":"sock","pid":1566,"zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"Successfully put SledAgentInfo to nexus","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:10.303273149Z","hostname":"sock","pid":1566,"component":"NexusNotifierTask","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/nexus.rs:500","new_generation":"1","old_generation":"1"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:10.633590223Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:47808"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:10.637998562Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"4c45cf73-6ff8-4597-a316-7f303c503659","remote_addr":"[fd00:1122:3344:101::c]:47808","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":3288,"response_code":"200"} {"msg":"wait for service Some(\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\") failed: Property not found. retry in 1.053380187s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:27:10.665932135Z","hostname":"sock","pid":1566,"zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:10.75800834Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:50994"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:10.759352216Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:10.762921262Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:10.762945713Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"4897cb49-8bf7-43a0-9925-3368f5f092c0","remote_addr":"[fd00:1122:3344:101::c]:47808","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":2118,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:10.866712613Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:38804"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:10.867305249Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"7702288d-205f-40ea-81c0-24259f7e3b01","remote_addr":"[fd00:1122:3344:101::b]:38804","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":75,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:10.914875683Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:47414"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:10.91659938Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"6b38ec8e-04ef-47ca-b2e5-5d0f111df03c","remote_addr":"[fd00:1122:3344:101::c]:47414","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":1369,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:10.927531795Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:10.927547045Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"e5dcdd4b-a2a5-46c6-bb5b-1d2a0f08d51c","remote_addr":"[fd00:1122:3344:101::b]:38804","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":52,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:10.966853886Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:47561"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:10.966970939Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:10.978082279Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"22d1e67f-40a6-46d0-8754-eefd49c7581b","remote_addr":"[fd00:1122:3344:101::c]:47808","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":3035,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:10.991022158Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:49746"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:10.991187002Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"b11c5eb8-4c9a-432c-a1d4-bceb43199868","remote_addr":"[fd00:1122:3344:101::b]:49746","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":97,"response_code":"200"} {"msg":"skipping delta compute for subnet","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:11.084750047Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:486","new_vers":"RouterVersion { router_id: 001de000-074c-4000-8000-000000000001, version: 4 }","old_vers":"RouterVersion { router_id: 001de000-074c-4000-8000-000000000001, version: 4 }","subnet":"RouterId { vni: Vni(100), kind: System }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:11.084816459Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"df38c27d-e964-401e-ba1c-f7ea9a669907","remote_addr":"[fd00:1122:3344:101::b]:38804","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":304,"response_code":"204"} {"msg":"wait for service Some(\"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76\") failed: Property not found. retry in 1.303418738s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:27:11.730227075Z","hostname":"sock","pid":1566,"zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.061299953Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_crucible_6ceaca86-5562-442f-a7a7-bbfd684c302b"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.061343574Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_crucible_25c723ea-c52b-44b6-8751-dbe925d1e7a7"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.061350034Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_nexus_a1e38c68-2155-4ce9-b70a-66d434b3f24f"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.061353914Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_nexus_743f2600-b4a7-4963-8bef-692428b6800a"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.061358005Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_clickhouse_40c9a9db-bd0a-45df-b2ea-f8a9ad278c76"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.061361545Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_crucible_pantry_100f148b-d0a5-4957-b7f4-7b142438883b"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.061365045Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_crucible_37091b30-13af-44d8-ac11-483eada5eeca"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.061368425Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_crucible_pantry_683f3bcd-05eb-48ca-a731-4591f3875c45"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.061371975Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_crucible_pantry_eb0487cd-afd0-473b-b56c-c84ebdb9a402"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.061377235Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_external_dns_7468547b-b2f3-4993-afd8-72e9164a4301"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.061380935Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_crucible_f12a4089-4d9a-4d0c-b3e6-8a12d533bb03"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.061384395Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_crucible_014247db-8886-4cf5-b68e-375be0dabc7f"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.061387815Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_oximeter_f71d33aa-7320-48b9-81d2-899a100e78cd"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.061401196Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_nexus_b4bce764-527e-4a0d-9efa-46f5760fbb1d"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.061404766Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_crucible_1ecfbd35-39cb-454a-b9bf-b046feb4f811"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.061408276Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_crucible_7361bad6-7bc4-4252-9897-84b3278d100e"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.061411706Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_external_dns_8b302e6b-b09c-4ef2-8768-1e5fae50d31a"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.061415176Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_crucible_6e9d55e6-6648-4adf-a2be-39839524cd72"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.061420626Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:3325","zone":"oxz_crucible_771664a2-5c9b-4063-8f72-848056c94344"} {"msg":"Writing ledger to /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.062339081Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.063089431Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.064351375Z","hostname":"sock","pid":1566,"uri":"/omicron-zones","method":"PUT","req_id":"700a055a-289e-4a32-b403-ee73c9f0bc71","remote_addr":"[fd00:1122:3344:101::1]:53729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":50387359,"response_code":"204"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.064374536Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.064959002Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.065815305Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.079975337Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.080024158Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_57cb94f0-4114-4bd3-970a-a0245b825f9d"} {"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.085554507Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1415"} {"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.089575206Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:790"} {"msg":"Nexus address: [fd00:1122:3344:101::c]:12221","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.094763896Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:803"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.097221282Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.097294564Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.102652259Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.113560313Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.113617465Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_27bb8d18-eb2e-4bb2-b35c-c031637c236f"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.130034208Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.130086459Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_57cb94f0-4114-4bd3-970a-a0245b825f9d"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.155478434Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.155545686Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_27bb8d18-eb2e-4bb2-b35c-c031637c236f"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.170332115Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.170388846Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_87e49e9a-4ab3-4574-b920-463506a8978e"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.186903362Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.18720861Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_87e49e9a-4ab3-4574-b920-463506a8978e"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.202513813Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.202547004Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_b471fd39-9806-47fa-8ddb-0ca2f42aaf5d"} {"msg":"rack_network_config: RackNetworkConfigV2 {\n bfd: [],\n bgp: [],\n infra_ip_first: 10.151.2.174,\n infra_ip_last: 10.151.2.174,\n ports: [\n PortConfigV2 {\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.2.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n autoneg: false,\n bgp_peers: [],\n lldp: None,\n port: \"qsfp0\",\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.2.100,\n rib_priority: None,\n vlan_id: None,\n },\n ],\n switch: Switch0,\n tx_eq: None,\n uplink_port_fec: Some(\n None,\n ),\n uplink_port_speed: Speed40G,\n },\n ],\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.208723171Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1017"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.226873591Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.226910992Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_03ea8739-b702-4a4b-b400-0745bd6b4f96"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.239367638Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.239416659Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_b471fd39-9806-47fa-8ddb-0ca2f42aaf5d"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.255815901Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.255861123Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_d8acd544-80aa-4e82-b375-eb038fe27547"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.272254745Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.272303116Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_03ea8739-b702-4a4b-b400-0745bd6b4f96"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.288178985Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.289766648Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_0aaa1466-164a-4540-ae41-ab17e9fc6bc8"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.303072247Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.303119198Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_d8acd544-80aa-4e82-b375-eb038fe27547"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.326102228Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.326153299Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_0aaa1466-164a-4540-ae41-ab17e9fc6bc8"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.339249823Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.339293454Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_56c8079e-8d4e-4506-b6ff-2f0a5efab320"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.356429367Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.356477588Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_56c8079e-8d4e-4506-b6ff-2f0a5efab320"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.372767677Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.373170378Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_276e9391-34d6-4a99-bd91-776ecada771e"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.388216554Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.388278886Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_276e9391-34d6-4a99-bd91-776ecada771e"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.400961208Z","hostname":"sock","pid":1566,"uri":"/inventory","method":"GET","req_id":"3b8a4faa-d969-4386-95e6-6e9d9b9f3730","remote_addr":"[fd00:1122:3344:101::b]:47561","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":2433672,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.401194075Z","hostname":"sock","pid":1566,"uri":"/inventory","method":"GET","req_id":"e2036eae-a184-4f64-8feb-e28f97c310eb","remote_addr":"[fd00:1122:3344:101::c]:50994","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":2642561,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.857378225Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:60778"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.857503168Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.857510258Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.857562349Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.857827466Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.871576628Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.871620909Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_57cb94f0-4114-4bd3-970a-a0245b825f9d"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.899504461Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.899551603Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_27bb8d18-eb2e-4bb2-b35c-c031637c236f"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.925719469Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.92576212Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_87e49e9a-4ab3-4574-b920-463506a8978e"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.9483674Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.948403831Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_b471fd39-9806-47fa-8ddb-0ca2f42aaf5d"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.969775577Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.969821779Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_03ea8739-b702-4a4b-b400-0745bd6b4f96"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.991405921Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:13.991446972Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_d8acd544-80aa-4e82-b375-eb038fe27547"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:14.013162448Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:14.01321119Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_0aaa1466-164a-4540-ae41-ab17e9fc6bc8"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:14.035940963Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:14.035984194Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_56c8079e-8d4e-4506-b6ff-2f0a5efab320"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:14.059033266Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:14.059090988Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_276e9391-34d6-4a99-bd91-776ecada771e"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:14.074511684Z","hostname":"sock","pid":1566,"uri":"/inventory","method":"GET","req_id":"d76b4ba4-85af-4b41-a59a-d0521094d823","remote_addr":"[fd00:1122:3344:101::b]:60778","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":217049,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:14.896805273Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:62389"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:14.899257108Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:767","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: b15378a9-5120-477b-abef-a170753103c1, kind: Service { id: 7468547b-b2f3-4993-afd8-72e9164a4301 }, name: Name(\"external-dns-7468547b-b2f3-4993-afd8-72e9164a4301\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 124])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: beea2870-fa6b-4d58-b0eb-af5a33149d11, kind: Service { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a }, name: Name(\"external-dns-8b302e6b-b09c-4ef2-8768-1e5fae50d31a\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 41])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 22195cd0-81ca-4c14-8294-e0bb8767fa51, kind: Service { id: 743f2600-b4a7-4963-8bef-692428b6800a }, name: Name(\"nexus-743f2600-b4a7-4963-8bef-692428b6800a\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 167, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 357305ec-a785-4a6e-96cf-879dc3168076, kind: Service { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f }, name: Name(\"nexus-a1e38c68-2155-4ce9-b70a-66d434b3f24f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 213])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 56f43fd0-1c39-449d-9089-5f10b7acf59b, kind: Service { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d }, name: Name(\"nexus-b4bce764-527e-4a0d-9efa-46f5760fbb1d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 31])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:14.89930851Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:14.899332621Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:14.899340191Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:14.899383242Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:14.899391672Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:14.899399863Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:14.899433193Z","hostname":"sock","pid":1566,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"d6c758b0-dd28-43c3-a298-bd8f5cde72ae","remote_addr":"[fd00:1122:3344:101::c]:62389","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":2529,"response_code":"204"} {"msg":"Handoff to Nexus is complete","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:14.900695967Z","hostname":"sock","pid":1566,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1089"} {"msg":"Writing ledger to /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:14.905064955Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:14.905335052Z","hostname":"sock","pid":1566,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:15.119517692Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:59262"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:15.119784679Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:767","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: b15378a9-5120-477b-abef-a170753103c1, kind: Service { id: 7468547b-b2f3-4993-afd8-72e9164a4301 }, name: Name(\"external-dns-7468547b-b2f3-4993-afd8-72e9164a4301\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 124])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: beea2870-fa6b-4d58-b0eb-af5a33149d11, kind: Service { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a }, name: Name(\"external-dns-8b302e6b-b09c-4ef2-8768-1e5fae50d31a\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 41])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 22195cd0-81ca-4c14-8294-e0bb8767fa51, kind: Service { id: 743f2600-b4a7-4963-8bef-692428b6800a }, name: Name(\"nexus-743f2600-b4a7-4963-8bef-692428b6800a\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 167, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 357305ec-a785-4a6e-96cf-879dc3168076, kind: Service { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f }, name: Name(\"nexus-a1e38c68-2155-4ce9-b70a-66d434b3f24f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 213])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 56f43fd0-1c39-449d-9089-5f10b7acf59b, kind: Service { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d }, name: Name(\"nexus-b4bce764-527e-4a0d-9efa-46f5760fbb1d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 31])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:15.119823271Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:15.119853911Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:15.119859541Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:15.119863352Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:15.119868262Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:15.119873842Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:15.119889422Z","hostname":"sock","pid":1566,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"2eed1fa2-f55c-4055-8171-adfc8019879f","remote_addr":"[fd00:1122:3344:101::c]:59262","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":251,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:15.173315003Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:56805"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:15.173447207Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:15.173460537Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:15.173505728Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:15.173807287Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:15.185273376Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:15.185312597Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_57cb94f0-4114-4bd3-970a-a0245b825f9d"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:15.207993229Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:15.20803339Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_27bb8d18-eb2e-4bb2-b35c-c031637c236f"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:15.231509713Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:15.231546414Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_87e49e9a-4ab3-4574-b920-463506a8978e"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:15.254005371Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:15.254059682Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_b471fd39-9806-47fa-8ddb-0ca2f42aaf5d"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:15.276333373Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:15.276379494Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_03ea8739-b702-4a4b-b400-0745bd6b4f96"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:15.297469164Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:15.297508454Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_d8acd544-80aa-4e82-b375-eb038fe27547"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:15.316759324Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:15.316796225Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_0aaa1466-164a-4540-ae41-ab17e9fc6bc8"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:15.334178084Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:15.334209975Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_56c8079e-8d4e-4506-b6ff-2f0a5efab320"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:15.354311957Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:15.354353608Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_276e9391-34d6-4a99-bd91-776ecada771e"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:15.365578121Z","hostname":"sock","pid":1566,"uri":"/inventory","method":"GET","req_id":"3d9f9695-d678-488f-9608-b3881703d872","remote_addr":"[fd00:1122:3344:101::c]:56805","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":192175,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:15.611651311Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:56953"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:15.61195059Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:767","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: b15378a9-5120-477b-abef-a170753103c1, kind: Service { id: 7468547b-b2f3-4993-afd8-72e9164a4301 }, name: Name(\"external-dns-7468547b-b2f3-4993-afd8-72e9164a4301\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 124])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: beea2870-fa6b-4d58-b0eb-af5a33149d11, kind: Service { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a }, name: Name(\"external-dns-8b302e6b-b09c-4ef2-8768-1e5fae50d31a\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 41])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 22195cd0-81ca-4c14-8294-e0bb8767fa51, kind: Service { id: 743f2600-b4a7-4963-8bef-692428b6800a }, name: Name(\"nexus-743f2600-b4a7-4963-8bef-692428b6800a\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 167, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 357305ec-a785-4a6e-96cf-879dc3168076, kind: Service { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f }, name: Name(\"nexus-a1e38c68-2155-4ce9-b70a-66d434b3f24f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 213])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 56f43fd0-1c39-449d-9089-5f10b7acf59b, kind: Service { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d }, name: Name(\"nexus-b4bce764-527e-4a0d-9efa-46f5760fbb1d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 31])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:15.612012701Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:15.612043392Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:15.612052622Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:15.612087993Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:15.612113284Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:15.612126054Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:15.612137265Z","hostname":"sock","pid":1566,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"2d20aa72-871e-4710-8a46-017aaafad602","remote_addr":"[fd00:1122:3344:101::b]:56953","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":363,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:16.502065118Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:39322"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:16.502314115Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:767","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: b15378a9-5120-477b-abef-a170753103c1, kind: Service { id: 7468547b-b2f3-4993-afd8-72e9164a4301 }, name: Name(\"external-dns-7468547b-b2f3-4993-afd8-72e9164a4301\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 124])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: beea2870-fa6b-4d58-b0eb-af5a33149d11, kind: Service { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a }, name: Name(\"external-dns-8b302e6b-b09c-4ef2-8768-1e5fae50d31a\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 41])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 22195cd0-81ca-4c14-8294-e0bb8767fa51, kind: Service { id: 743f2600-b4a7-4963-8bef-692428b6800a }, name: Name(\"nexus-743f2600-b4a7-4963-8bef-692428b6800a\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 167, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 357305ec-a785-4a6e-96cf-879dc3168076, kind: Service { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f }, name: Name(\"nexus-a1e38c68-2155-4ce9-b70a-66d434b3f24f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 213])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 56f43fd0-1c39-449d-9089-5f10b7acf59b, kind: Service { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d }, name: Name(\"nexus-b4bce764-527e-4a0d-9efa-46f5760fbb1d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 31])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:16.502367726Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:16.502407948Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:16.502428148Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:16.502442588Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:16.502465239Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:16.50249152Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:16.502524001Z","hostname":"sock","pid":1566,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"df779ed1-341b-4ffa-806c-ea50ea4242be","remote_addr":"[fd00:1122:3344:101::a]:39322","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":340,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:21.605259032Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:33658"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:21.605428166Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"d3340157-1b43-4392-bdd6-8f2ddf3a81a9","remote_addr":"[fd00:1122:3344:101::c]:33658","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":84,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:21.654751227Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:21.654775488Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"e20dc432-adb1-458f-950c-ac94d5535b98","remote_addr":"[fd00:1122:3344:101::c]:33658","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":72,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:21.666203636Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"c1adb27e-be99-4480-a6d7-b7109e6ecd1d","remote_addr":"[fd00:1122:3344:101::c]:33658","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":185,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:38.819770212Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:63910"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:38.82046721Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"2b1d6897-d057-4da7-ba80-32e83d8fdedb","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:63910","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":51,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:39.844784971Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:39091"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:39.844935605Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"1e02a92b-597b-45b8-8d2e-0d5023b4b5a9","remote_addr":"[fd00:1122:3344:101::a]:39091","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":76,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:39.918335856Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:39.918361996Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"f501666a-c8c3-45b0-be38-d144b5a4f75f","remote_addr":"[fd00:1122:3344:101::a]:39091","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":84,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:39.950540595Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:52459"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:39.95072315Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"726f91d4-6a50-4d1d-8cfc-6f18eceed0a3","remote_addr":"[fd00:1122:3344:101::a]:52459","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":134,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:39.958477639Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"5edbe625-3c3a-4610-aeeb-8c61e0760b91","remote_addr":"[fd00:1122:3344:101::a]:39091","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":157,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:40.003972067Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:45973"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:40.00520542Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:40.087694536Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:40.148889727Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"d4564b65-6049-42fa-95a8-c0b5bbcf6771","remote_addr":"[fd00:1122:3344:101::a]:45973","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":144870,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:40.152622368Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"92285650-d22a-4bd0-955c-ab94a1fc803f","remote_addr":"[fd00:1122:3344:101::a]:45973","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":1524,"response_code":"200"} {"msg":"Attempting to update network config with generation: 2, current_generation: 1","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:40.213086809Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:40.214210929Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:40.215289068Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 2","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:40.21647294Z","hostname":"sock","pid":1566,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:40.216934693Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"PUT","req_id":"4ae4be73-e4b6-43a0-9b60-d51157d441f5","remote_addr":"[fd00:1122:3344:101::a]:45973","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":5414,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:40.636730451Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:37932"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:40.636845474Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"f6365db1-cc99-4a7b-b920-4e88cbcc587d","remote_addr":"[fd00:1122:3344:101::b]:37932","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":50,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:40.64855899Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:63317"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:40.648673173Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"c8da66b1-b31c-4998-8643-e8320a5a6780","remote_addr":"[fd00:1122:3344:101::c]:63317","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":44,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:40.663725229Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:40.66375747Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"fcaeed0e-9dac-47d9-bdd3-32d694585bc9","remote_addr":"[fd00:1122:3344:101::b]:37932","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":80,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:40.680825471Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"cd28bded-b4c5-496f-a064-bc848a4e8164","remote_addr":"[fd00:1122:3344:101::b]:37932","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":227,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:40.723825781Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:56923"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:40.724120479Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"02f07882-a361-4982-9b8b-ac5e7b879fcf","remote_addr":"[fd00:1122:3344:101::c]:56923","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":190,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:40.727016127Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:40.727042938Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"a2fd1c63-678d-4fdb-9d90-ec980bbd22aa","remote_addr":"[fd00:1122:3344:101::c]:63317","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":80,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:40.735753403Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"56121692-1a72-45a5-bf3c-a35ae4b7a04c","remote_addr":"[fd00:1122:3344:101::c]:63317","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":151,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:40.802898475Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:36571"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:40.8031045Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"e3861a55-6652-41f4-bfc4-68e398403e8b","remote_addr":"[fd00:1122:3344:101::b]:36571","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":129,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:40.843754727Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:44866"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:40.843838919Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:40.880814877Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:40.936748147Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"8a0e4753-f6b3-4ba6-a6f5-c91b52c310f6","remote_addr":"[fd00:1122:3344:101::c]:44866","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":92953,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:40.936912211Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:41073"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:40.937040294Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:40.940101727Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"09d63e4a-1d5e-4981-bda1-448cf5153434","remote_addr":"[fd00:1122:3344:101::c]:44866","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":423,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:40.970656412Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:41.025634685Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"fb72e83c-a06e-407b-ba61-ab9a5ed56db7","remote_addr":"[fd00:1122:3344:101::b]:41073","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":88647,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:27:41.026076237Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"eb13e59f-a4ae-495e-87f1-7c2958884aff","remote_addr":"[fd00:1122:3344:101::b]:41073","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":155,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:00.752921964Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:63949","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","component":"producer-server","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::d]:56294"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:00.834961808Z","hostname":"sock","pid":1566,"uri":"/ce66c06a-7d54-47a6-860d-59da68080fee","method":"GET","req_id":"1a3b0a7e-22d6-421e-867d-4ee9c5655d22","remote_addr":"[fd00:1122:3344:101::d]:56294","local_addr":"[fd00:1122:3344:101::1]:63949","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":81805,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:08.853622184Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:35993"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:08.853781639Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"e439b6e1-6d06-4e9a-8c35-dac006982ea3","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:35993","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":66,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:09.92298446Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:61135"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:09.923173725Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"56e0b4bb-cceb-414c-8c04-16a9b7808210","remote_addr":"[fd00:1122:3344:101::a]:61135","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":85,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:09.982938588Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:09.982970389Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"6b53789a-c8f2-4f2f-8935-3e6133104bfd","remote_addr":"[fd00:1122:3344:101::a]:61135","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":100,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:09.998448537Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"1e15c426-5e11-4a09-8002-f93f9848595a","remote_addr":"[fd00:1122:3344:101::a]:61135","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":235,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:10.08270486Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:53211"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:10.082818804Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:10.119963226Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:10.178028132Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"dcc1f12a-e0a4-4bca-8711-658acc9e72c7","remote_addr":"[fd00:1122:3344:101::a]:53211","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":95256,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:10.178391812Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"39d2a682-95a0-4d0d-b438-e03d40ff8305","remote_addr":"[fd00:1122:3344:101::a]:53211","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":124,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:10.198373751Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:36941"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:10.198604307Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"8eed5314-089b-4a48-a775-15c5551d1489","remote_addr":"[fd00:1122:3344:101::a]:36941","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":145,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:10.647258414Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:54348"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:10.647361717Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"95be1b2b-c8ab-4563-b8ac-b421be827ada","remote_addr":"[fd00:1122:3344:101::c]:54348","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":41,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:10.763938152Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:60863"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:10.764045195Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"248e6b19-b251-41e1-99df-64193269743c","remote_addr":"[fd00:1122:3344:101::b]:60863","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":56,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:10.767756015Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:10.767775806Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"520dde5e-c52f-49fe-b3c6-48e1d5988095","remote_addr":"[fd00:1122:3344:101::c]:54348","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":67,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:10.780541291Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"dca37714-178d-46e2-849d-49ce8bf58ac9","remote_addr":"[fd00:1122:3344:101::c]:54348","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":163,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:10.809311797Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:10.809325487Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"309c7b7a-8746-44fd-9130-970de333034c","remote_addr":"[fd00:1122:3344:101::b]:60863","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":30,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:10.820696874Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"1ee8f51c-de6b-4677-b87f-327af4d4d20e","remote_addr":"[fd00:1122:3344:101::b]:60863","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":159,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:10.870496238Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:34972"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:10.870685063Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"e95bbcb8-c7ac-41d1-ad67-804966cdd158","remote_addr":"[fd00:1122:3344:101::c]:34972","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":107,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:10.87279561Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:42086"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:10.872861272Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:10.911443203Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:10.968677997Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"9295388c-488a-43db-9784-6ecfadfb20a4","remote_addr":"[fd00:1122:3344:101::c]:42086","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":95845,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:10.968979925Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"aaf5d894-cf90-4cb3-a51e-41b6e0d87f41","remote_addr":"[fd00:1122:3344:101::c]:42086","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":103,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:11.058504571Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:65249"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:11.058729467Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"cd8663d4-2641-414e-a48b-09ea52696c26","remote_addr":"[fd00:1122:3344:101::b]:65249","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":163,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:11.145128919Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:36217"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:11.145246372Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:11.182459786Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:11.240313447Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"7c575d8a-9ba4-4be1-96a4-1a574f0f46fe","remote_addr":"[fd00:1122:3344:101::b]:36217","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":95126,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:11.240568764Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"2b2425c5-a12f-421d-89a7-bfe104513f08","remote_addr":"[fd00:1122:3344:101::b]:36217","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":98,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:26.961653904Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:30.758731565Z","hostname":"sock","pid":1566,"uri":"/ce66c06a-7d54-47a6-860d-59da68080fee","method":"GET","req_id":"c135dc3a-287a-4090-8b74-872b35365357","remote_addr":"[fd00:1122:3344:101::d]:56294","local_addr":"[fd00:1122:3344:101::1]:63949","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":6633,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:38.887572125Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:51648"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:38.888197562Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"8595098b-a245-4e64-9345-b0b03c56f0ec","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:51648","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":50,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:39.84759421Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:33150"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:39.847729584Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"4effba25-a1bb-4b4c-a091-635547eb8fa4","remote_addr":"[fd00:1122:3344:101::a]:33150","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":83,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:39.890372375Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:39.890394975Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"30e8b312-1a27-457a-966b-005c0c93c4d7","remote_addr":"[fd00:1122:3344:101::a]:33150","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":73,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:39.898850093Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"76465858-a487-4b5a-b2e4-21983dec6056","remote_addr":"[fd00:1122:3344:101::a]:33150","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":203,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:39.906834679Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:41753"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:39.906994033Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"3fa7cc3b-4a21-44f1-b566-b5c944a2862a","remote_addr":"[fd00:1122:3344:101::a]:41753","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":108,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:39.984522515Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:49324"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:39.98468814Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:40.019553901Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:40.07290809Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"2176d486-838c-44f7-867f-95ffa09d909c","remote_addr":"[fd00:1122:3344:101::a]:49324","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":88292,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:40.073212099Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"190f70e8-594d-416e-a03d-c8ca5e717b9a","remote_addr":"[fd00:1122:3344:101::a]:49324","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":123,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:40.610041864Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:62956"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:40.610140487Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"6efaafbd-fda6-4e0c-9ebf-01bc07ba1fed","remote_addr":"[fd00:1122:3344:101::c]:62956","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":57,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:40.661651107Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:40.661676008Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"dd1fd8fe-31f0-4cfb-a27d-c6966ec2e875","remote_addr":"[fd00:1122:3344:101::c]:62956","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":65,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:40.673893707Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"3a871601-2540-47f1-99f3-59edaaf0b6f1","remote_addr":"[fd00:1122:3344:101::c]:62956","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":169,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:40.697026591Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:55737"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:40.697168755Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:40.734112442Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:40.785313834Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"0103a1d4-4ad9-4dff-a407-5991a53740e8","remote_addr":"[fd00:1122:3344:101::c]:55737","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":88208,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:40.785444457Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:40720"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:40.785473848Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:43848"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:40.785644453Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"4cbc86fd-33f1-41ce-b430-c68fd099f0f3","remote_addr":"[fd00:1122:3344:101::b]:43848","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":94,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:40.785840058Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"3bd37476-771d-46d6-a738-78f2ad915a9c","remote_addr":"[fd00:1122:3344:101::c]:55737","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":260,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:40.785960241Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"cf5cc291-0b18-4548-9b68-d869124a3b72","remote_addr":"[fd00:1122:3344:101::c]:40720","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":209,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:40.812176969Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:46676"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:40.812380144Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"8d94fdc6-44c3-4e89-8727-8cdbccab52d2","remote_addr":"[fd00:1122:3344:101::b]:46676","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":126,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:40.837161653Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:40.837181293Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"864a1c06-cd6c-42d8-96ac-dbd9a87340f1","remote_addr":"[fd00:1122:3344:101::b]:43848","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":67,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:40.846817533Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"d6b7140d-6ff9-4fd9-845a-7dfa74bb626f","remote_addr":"[fd00:1122:3344:101::b]:43848","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":230,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:40.8870973Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:55404"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:40.887206203Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:40.921685764Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:40.974986192Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"395131ec-6bcf-4099-8b11-307438628dbc","remote_addr":"[fd00:1122:3344:101::b]:55404","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":87821,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:28:40.975260879Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"c5682fb3-6692-4c44-b1ef-a9d3d9a74195","remote_addr":"[fd00:1122:3344:101::b]:55404","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":114,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:00.757135777Z","hostname":"sock","pid":1566,"uri":"/ce66c06a-7d54-47a6-860d-59da68080fee","method":"GET","req_id":"132ebd40-36c7-4725-9e75-1f753ed37c51","remote_addr":"[fd00:1122:3344:101::d]:56294","local_addr":"[fd00:1122:3344:101::1]:63949","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":5967,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:08.927265581Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:50901"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:08.927394695Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"b1f4169e-fa01-4e14-9cbd-4a9342c71482","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:50901","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":63,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:10.065169007Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:62988"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:10.065306731Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"80dbe23f-c368-44b1-bbb9-4046c27877de","remote_addr":"[fd00:1122:3344:101::a]:62988","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":82,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:10.10865672Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:10.108686951Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"ae14fef1-e056-4772-90fd-7bdc9b4c9fb3","remote_addr":"[fd00:1122:3344:101::a]:62988","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":77,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:10.117745696Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"92141a75-3838-4a5a-931a-fd878c1211a4","remote_addr":"[fd00:1122:3344:101::a]:62988","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":197,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:10.185526954Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:54927"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:10.185687029Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"5b2df19f-e0c0-4ad4-bcbd-b2b2de75f726","remote_addr":"[fd00:1122:3344:101::a]:54927","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":120,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:10.209004008Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:39434"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:10.20910358Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:10.250010694Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:10.307603018Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"575d58a9-9d84-43ff-ad4a-3342796ebf31","remote_addr":"[fd00:1122:3344:101::a]:39434","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":98546,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:10.307894926Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"c9e59873-3219-4a76-ae1c-8992559dce49","remote_addr":"[fd00:1122:3344:101::a]:39434","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":107,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:10.651900289Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:37327"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:10.652034813Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"953494f0-04ae-44bd-bd64-9be3ffe06304","remote_addr":"[fd00:1122:3344:101::c]:37327","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":45,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:10.712572836Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:10.712597237Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"92ac01f2-3af7-4ae9-af13-b13a8241cd18","remote_addr":"[fd00:1122:3344:101::c]:37327","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":64,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:10.726678027Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"e3be282c-b566-4e1e-9696-614f152be499","remote_addr":"[fd00:1122:3344:101::c]:37327","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":177,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:10.782410551Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:35582"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:10.782566695Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"a383d840-2ffc-4bf3-b873-995bab47107b","remote_addr":"[fd00:1122:3344:101::c]:35582","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":115,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:10.821490406Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:58770"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:10.821610939Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"657ee38c-a837-48b4-935a-3545f05ae3b1","remote_addr":"[fd00:1122:3344:101::b]:58770","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":32,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:10.834839716Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:38446"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:10.835052012Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:10.873506079Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:10.926637883Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"1cf5648c-cfda-4ca6-b43c-d1f18c3a3702","remote_addr":"[fd00:1122:3344:101::c]:38446","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":91686,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:10.926990792Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:10.927031123Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"5070b363-81cd-4f32-8f39-d27cae36871e","remote_addr":"[fd00:1122:3344:101::b]:58770","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":155,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:10.927106976Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"526f0ab3-c756-4767-b6a0-67acfa236a99","remote_addr":"[fd00:1122:3344:101::c]:38446","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":234,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:10.938508253Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"5a07385e-d74b-4070-a275-64e08003fcbe","remote_addr":"[fd00:1122:3344:101::b]:58770","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":257,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:11.091332987Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:47354"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:11.091511442Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"9eb233de-00be-4bfd-b6d7-a86332b283d0","remote_addr":"[fd00:1122:3344:101::b]:47354","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":122,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:11.149761314Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:57576"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:11.149866797Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:11.186345171Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:11.242113485Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"6bf84274-14aa-4a19-b20c-4c019881f692","remote_addr":"[fd00:1122:3344:101::b]:57576","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":92286,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:11.242435724Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"5f03f442-e6ff-4862-9f24-7e1306ba1950","remote_addr":"[fd00:1122:3344:101::b]:57576","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":104,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:30.756591859Z","hostname":"sock","pid":1566,"uri":"/ce66c06a-7d54-47a6-860d-59da68080fee","method":"GET","req_id":"5624f7fd-2641-477a-b574-209d8518afb5","remote_addr":"[fd00:1122:3344:101::d]:56294","local_addr":"[fd00:1122:3344:101::1]:63949","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":5554,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:38.962760206Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:43112"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:38.962887319Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"73578598-bef3-465e-9071-3f390829f5b9","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:43112","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":65,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:39.933347696Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:58159"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:39.93347901Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"2a1323cc-b0ba-45e6-8281-bd6d5ebb5d4b","remote_addr":"[fd00:1122:3344:101::a]:58159","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":81,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:39.947756305Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:39.947775756Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"e14fee3f-7044-4b41-97a2-1b90759fffe5","remote_addr":"[fd00:1122:3344:101::a]:58159","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":75,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:39.956191263Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"27052c10-2621-4eda-af3b-a872f65f683e","remote_addr":"[fd00:1122:3344:101::a]:58159","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":259,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:39.994160277Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:39901"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:39.994272391Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:40.034903647Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:40.09172256Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"2f2fba43-69ae-464e-9e63-49bc564c8ccd","remote_addr":"[fd00:1122:3344:101::a]:39901","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":97494,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:40.091894755Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:44730"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:40.09209942Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"f6a991ab-be0f-4e18-a369-125bb71843e2","remote_addr":"[fd00:1122:3344:101::a]:39901","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":167,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:40.092109971Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"674983cf-a80d-4f16-972a-9844b7373bb9","remote_addr":"[fd00:1122:3344:101::a]:44730","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":146,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:40.622875463Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:61068"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:40.623013226Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"6ab6ce34-67fa-4ac8-810a-dbb44d340115","remote_addr":"[fd00:1122:3344:101::c]:61068","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":80,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:40.639515612Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:40.639529892Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"e3f8ed62-1e2d-4bc9-a631-3cd49fe36933","remote_addr":"[fd00:1122:3344:101::c]:61068","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":42,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:40.653063777Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"4bf7d106-5000-4a0f-91ab-485e0f7d16cd","remote_addr":"[fd00:1122:3344:101::c]:61068","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":152,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:40.700741884Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:44439"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:40.700836306Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"9acd6db1-b48b-49cc-b547-fce3eb561cfe","remote_addr":"[fd00:1122:3344:101::b]:44439","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":26,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:40.712918632Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:55113"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:40.713089307Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"5e6583a9-e263-4056-a6b6-06a374064734","remote_addr":"[fd00:1122:3344:101::c]:55113","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":98,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:40.718506853Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:52091"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:40.718569515Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:40.755709357Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:40.812462938Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"6153302b-b57c-4dc5-ab71-f5346b94ce58","remote_addr":"[fd00:1122:3344:101::c]:52091","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":93912,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:40.812650624Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:63857"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:40.812691305Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:40.812720796Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"ae37e163-b806-4ebe-954d-c726779252ab","remote_addr":"[fd00:1122:3344:101::b]:44439","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":110,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:40.812844339Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"fc25c38b-5393-4dca-addf-8647c7ab2f0e","remote_addr":"[fd00:1122:3344:101::c]:52091","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":184,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:40.812932031Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"0c28c591-2d77-4412-bf46-7b16ba2b2bc8","remote_addr":"[fd00:1122:3344:101::b]:63857","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":190,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:40.821329998Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"fd38ce32-5f7a-4501-a137-0381b6148d93","remote_addr":"[fd00:1122:3344:101::b]:44439","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":178,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:40.824694479Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:62774"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:40.824772351Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:40.858785688Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:40.914135562Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"e0514de3-a549-4e7b-90f0-2a9d53f91d47","remote_addr":"[fd00:1122:3344:101::b]:62774","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":89388,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:29:40.914397639Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"3922576b-bf84-4a1a-bbb9-edf45afb6820","remote_addr":"[fd00:1122:3344:101::b]:62774","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":104,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:00.75788819Z","hostname":"sock","pid":1566,"uri":"/ce66c06a-7d54-47a6-860d-59da68080fee","method":"GET","req_id":"8fd17001-22c0-4640-9682-ba88fd548a02","remote_addr":"[fd00:1122:3344:101::d]:56294","local_addr":"[fd00:1122:3344:101::1]:63949","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":6005,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:08.996698838Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:49718"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:08.996816901Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"9dc3d348-9c2d-4e32-9001-dc87f86940bb","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:49718","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":55,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:09.936838097Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:40138"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:09.936972971Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"b2a167dc-9d32-47f1-a455-a70f47f89234","remote_addr":"[fd00:1122:3344:101::a]:40138","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":69,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:09.983689301Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:09.983726562Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"fd5fde6d-b9e2-46d9-be5b-1fbe5af7c40a","remote_addr":"[fd00:1122:3344:101::a]:40138","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":82,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:09.993461135Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"f20ffaae-d2e9-4517-8064-7f52a6b42a05","remote_addr":"[fd00:1122:3344:101::a]:40138","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":216,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:10.061538822Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:49105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:10.061731917Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"15a37bc7-bff6-465b-9e23-e8b72e9471f5","remote_addr":"[fd00:1122:3344:101::a]:49105","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":133,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:10.198047005Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:45435"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:10.198162708Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:10.237403857Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:10.293611374Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"cb5cfd35-7ae4-4aa1-9707-072912628a38","remote_addr":"[fd00:1122:3344:101::a]:45435","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":95504,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:10.293964913Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"893b11d4-e3de-44cd-b679-b83d3411393b","remote_addr":"[fd00:1122:3344:101::a]:45435","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":107,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:10.684908733Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:56823"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:10.685028586Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"0334f7f5-1eea-498d-adb4-fd7819df019c","remote_addr":"[fd00:1122:3344:101::c]:56823","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":53,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:10.712450976Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:10.712489227Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"8a6910cd-2c0c-43fe-8139-6b5642a32b90","remote_addr":"[fd00:1122:3344:101::c]:56823","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":73,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:10.727573164Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"c8a5f89a-ead4-4614-95a8-016394d03396","remote_addr":"[fd00:1122:3344:101::c]:56823","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":266,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:10.81414623Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:38432"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:10.814249343Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"d0b3b25b-6db2-4b6c-97ac-88a9b99e0328","remote_addr":"[fd00:1122:3344:101::b]:38432","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":59,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:10.857807479Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:10.8578414Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"df2d941a-777e-4b70-9096-004c8766c988","remote_addr":"[fd00:1122:3344:101::b]:38432","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":69,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:10.868205639Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"a216e6d6-6ba6-4a90-a76e-2e7cda2c0063","remote_addr":"[fd00:1122:3344:101::b]:38432","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":225,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:10.915193957Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:50084"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:10.915992719Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"6e0212ef-4a1e-4154-8ad3-b802e7588c20","remote_addr":"[fd00:1122:3344:101::b]:50084","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":185,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:10.930190302Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:45488"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:10.930281234Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:10.969191104Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:11.026546612Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"f7b505f3-397c-4682-9baa-bb79f0e15df7","remote_addr":"[fd00:1122:3344:101::b]:45488","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":96312,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:11.026790919Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"29a88133-b595-49dd-81d1-7ea1535dc8a2","remote_addr":"[fd00:1122:3344:101::b]:45488","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":109,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:11.124566417Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:33227"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:11.124787383Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"d2198823-c73e-49ab-a35c-246ad504e27a","remote_addr":"[fd00:1122:3344:101::c]:33227","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":148,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:11.12765163Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:63541"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:11.127775104Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:11.164595787Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:11.221381639Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"570edb9d-ec6a-42dc-bec0-cf93d6d9b876","remote_addr":"[fd00:1122:3344:101::c]:63541","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":93653,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:11.221621955Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"8c57e4ab-358e-4f4c-8b64-792fcde0b67f","remote_addr":"[fd00:1122:3344:101::c]:63541","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":94,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:30.75727987Z","hostname":"sock","pid":1566,"uri":"/ce66c06a-7d54-47a6-860d-59da68080fee","method":"GET","req_id":"3e76e70f-c65b-4060-b1f4-818c166d340c","remote_addr":"[fd00:1122:3344:101::d]:56294","local_addr":"[fd00:1122:3344:101::1]:63949","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":6875,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:39.030621759Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:60886"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:39.030738803Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"6e24ee68-892f-4723-b2ca-091e50731458","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:60886","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":53,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:39.741351808Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:46858"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:39.741486962Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"17496b9d-76bf-46f2-b1d4-3d6249614f28","remote_addr":"[fd00:1122:3344:101::a]:46858","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":72,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:39.796902877Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:39.796928458Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"c47d397f-b70a-4603-8722-67f726bd4410","remote_addr":"[fd00:1122:3344:101::a]:46858","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":83,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:39.804700767Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"79cf6d6f-32fe-4ce3-9975-58834c68bae5","remote_addr":"[fd00:1122:3344:101::a]:46858","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":239,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:39.875800256Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:47252"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:39.875978441Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"223af7c6-89cf-4c0e-899d-5b50efc9f2f9","remote_addr":"[fd00:1122:3344:101::a]:47252","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":131,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:39.906769512Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:63107"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:39.906862034Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:39.948266451Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:40.006218835Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"4b66d6a7-1a65-4a66-b79f-7daab907fd51","remote_addr":"[fd00:1122:3344:101::a]:63107","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":99390,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:40.006578095Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"4781202a-5e57-406f-adb0-5eafb368291a","remote_addr":"[fd00:1122:3344:101::a]:63107","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":148,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:40.625843915Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:44449"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:40.625941578Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"e0828423-07ca-40ea-aaa5-58fb99821150","remote_addr":"[fd00:1122:3344:101::c]:44449","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":56,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:40.633473991Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:34858"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:40.633547313Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"076c0eaa-142d-44a0-9f40-2cafd34319a5","remote_addr":"[fd00:1122:3344:101::b]:34858","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":21,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:40.696947054Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:40.696973414Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"9310a663-72b5-4ee4-beb0-3f041a17e354","remote_addr":"[fd00:1122:3344:101::b]:34858","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":79,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:40.69756007Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:40.697568371Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"196bff5a-16b4-4c44-9ce4-7bbaca79248e","remote_addr":"[fd00:1122:3344:101::c]:44449","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":20,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:40.706843131Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"07f6fd3a-5d57-45e5-9587-0283ba5ef1d0","remote_addr":"[fd00:1122:3344:101::b]:34858","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":164,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:40.708190597Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"f267205e-a0c7-46b2-b3ff-ffdcd3ab8b97","remote_addr":"[fd00:1122:3344:101::c]:44449","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":81,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:40.75870493Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:60486"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:40.758858175Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"437dd059-f13e-4210-8a32-35f442c3d20c","remote_addr":"[fd00:1122:3344:101::c]:60486","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":90,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:40.773821818Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:49498"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:40.7738836Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:40.813592981Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:40.86875141Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"af3554ee-89c8-4b77-a56e-2b2dfeae2266","remote_addr":"[fd00:1122:3344:101::c]:49498","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":94889,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:40.868851743Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:37573"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:40.868872103Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:35588"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:40.869141211Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"9f0b1c48-d982-41d8-9dbe-c2867b882cc0","remote_addr":"[fd00:1122:3344:101::b]:35588","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":188,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:40.869153341Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:40.872808369Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"742a2451-00d9-4e1a-a918-6a04cf8ced24","remote_addr":"[fd00:1122:3344:101::c]:49498","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":3738,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:40.904469624Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:40.960703961Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"27cbe932-2bec-4375-a51b-ae7b4c8bd093","remote_addr":"[fd00:1122:3344:101::b]:37573","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":91665,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:30:40.960968968Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"a19a7371-7a65-41fb-b8c7-4214e7e97eee","remote_addr":"[fd00:1122:3344:101::b]:37573","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":117,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:00.756881025Z","hostname":"sock","pid":1566,"uri":"/ce66c06a-7d54-47a6-860d-59da68080fee","method":"GET","req_id":"74d3baec-e1be-497f-a7f2-05394ba401c8","remote_addr":"[fd00:1122:3344:101::d]:56294","local_addr":"[fd00:1122:3344:101::1]:63949","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":5766,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:09.065049325Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:47661"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:09.065173228Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"5b7425f4-ecd3-4bc6-ab79-bb74c6c9cd75","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:47661","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":67,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:09.944692911Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:36365"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:09.944836405Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"09707c32-b9b7-4acf-a5df-6238dac4f818","remote_addr":"[fd00:1122:3344:101::a]:36365","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":88,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:09.991632958Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:09.991652758Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"57f7b519-01c3-4ed5-95d0-449e7c93ce81","remote_addr":"[fd00:1122:3344:101::a]:36365","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":82,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:09.999451229Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"ed95efd6-631d-45d4-96c7-877bc47cd5da","remote_addr":"[fd00:1122:3344:101::a]:36365","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":218,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:10.084055482Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:52943"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:10.084224957Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"5f894792-edec-446d-b04d-4e333569b855","remote_addr":"[fd00:1122:3344:101::a]:52943","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":112,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:10.224782609Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:56393"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:10.224889152Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:10.267141472Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:10.32155701Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"0919df81-0e17-43ab-90ed-729588b91a83","remote_addr":"[fd00:1122:3344:101::a]:56393","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":96715,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:10.321848288Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"b0479842-0ffa-4b51-8a51-0757d9dfa489","remote_addr":"[fd00:1122:3344:101::a]:56393","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":93,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:10.700898897Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:62819"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:10.700977389Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"f7564e3e-a459-4510-a884-96f97890f4d4","remote_addr":"[fd00:1122:3344:101::c]:62819","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":38,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:10.743913998Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:41864"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:10.744023381Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"6b9df8c9-b44f-4063-a160-cc4af839e7ff","remote_addr":"[fd00:1122:3344:101::b]:41864","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":21,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:10.793865706Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:10.793916027Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"c4fb5824-6a12-4151-bffe-d8b66c348a10","remote_addr":"[fd00:1122:3344:101::b]:41864","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":105,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:10.806842266Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"d6c4faa3-039f-445a-9cc5-e2a917c3884e","remote_addr":"[fd00:1122:3344:101::b]:41864","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":178,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:10.877300787Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:48656"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:10.877477692Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"9aaa6e77-1104-433d-9be2-9a0a215aaf82","remote_addr":"[fd00:1122:3344:101::c]:48656","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":122,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:10.922147897Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:10.922169398Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"f8214d69-7610-477f-8560-b56e4a10e7b1","remote_addr":"[fd00:1122:3344:101::c]:62819","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":54,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:10.930284527Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"ba19d75c-602c-4565-a4e4-a727be2f5904","remote_addr":"[fd00:1122:3344:101::c]:62819","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":155,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:10.938474508Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:46889"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:10.938592821Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"22aeaa7d-86d3-4cf2-bb34-81f7f7ba2912","remote_addr":"[fd00:1122:3344:101::b]:46889","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":85,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:10.940966195Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:63962"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:10.941050667Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:10.981041726Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:11.038661671Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"3b002c93-981d-4781-b65c-f11c2f9ed03e","remote_addr":"[fd00:1122:3344:101::b]:63962","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":97641,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:11.038779704Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:54723"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:11.03897741Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:11.042540536Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"317b72d8-3576-42ef-9448-cc1b03b4494d","remote_addr":"[fd00:1122:3344:101::b]:63962","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":3573,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:11.073191253Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:11.13201312Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"5e18a6cc-8293-4f51-a7ee-869903c7eec3","remote_addr":"[fd00:1122:3344:101::c]:54723","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":93114,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:11.132298218Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"44084922-6325-48a1-8db1-790364523e47","remote_addr":"[fd00:1122:3344:101::c]:54723","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":119,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:30.756134142Z","hostname":"sock","pid":1566,"uri":"/ce66c06a-7d54-47a6-860d-59da68080fee","method":"GET","req_id":"40bbe16c-9be5-4fb3-b728-44ddfec315c7","remote_addr":"[fd00:1122:3344:101::d]:56294","local_addr":"[fd00:1122:3344:101::1]:63949","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":5630,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:39.102517103Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:35125"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:39.102632926Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"5039618e-419b-4432-a49e-c0a3e1da8b1d","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:35125","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":60,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:39.952506859Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:42559"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:39.952642762Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"a99bd7a9-7e4e-4f91-8d0c-9967d92e2959","remote_addr":"[fd00:1122:3344:101::a]:42559","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":86,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:39.964544503Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:57434"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:39.964736739Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"a80d32ef-ff75-436e-a285-2dbd14c93720","remote_addr":"[fd00:1122:3344:101::a]:57434","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":130,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:39.998460189Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:39.998473649Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"fa30cf31-ec1e-4e61-a8c3-2f3c61eb77db","remote_addr":"[fd00:1122:3344:101::a]:42559","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":60,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:40.005988732Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:53602"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:40.006076214Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:40.045422176Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:40.102162937Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"10f21878-e03d-410d-9781-cb15b706e4f2","remote_addr":"[fd00:1122:3344:101::a]:53602","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":96125,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:40.102487166Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"d8219a94-5965-4832-85e6-b5354bd8bbc0","remote_addr":"[fd00:1122:3344:101::a]:53602","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":154,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:40.102725532Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"42aeeb77-e3ca-41e5-8285-844c86f3fbbd","remote_addr":"[fd00:1122:3344:101::a]:42559","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":331,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:40.59737109Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:51506"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:40.597508543Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"3ec7e1fb-bb1f-402c-8250-e13a02964e2b","remote_addr":"[fd00:1122:3344:101::c]:51506","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":59,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:40.623437903Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:40.623464464Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"e62971c8-e5b0-48a6-82e2-944cee302151","remote_addr":"[fd00:1122:3344:101::c]:51506","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":75,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:40.669098595Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:57744"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:40.669419354Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"a95abf46-99d7-4123-ab23-4d44baee669c","remote_addr":"[fd00:1122:3344:101::c]:57744","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":201,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:40.671838249Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"8a1b366b-c658-40bb-b2d0-b64e431c6741","remote_addr":"[fd00:1122:3344:101::c]:51506","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":299,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:40.768943629Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:48229"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:40.769041502Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:40.809317829Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:40.867790157Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"37be69d2-8694-43a6-bbcd-a30d118e4b69","remote_addr":"[fd00:1122:3344:101::c]:48229","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":98786,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:40.867977442Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:47104"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:40.868170677Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"1e9b35a4-e383-46d0-8521-b733d6891620","remote_addr":"[fd00:1122:3344:101::b]:47104","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":99,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:40.86826361Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"1a11d774-8b41-49ec-a288-78cbe084bd39","remote_addr":"[fd00:1122:3344:101::c]:48229","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":236,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:40.883624364Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:40.883642705Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"620db641-e195-42a9-978e-de36404f117d","remote_addr":"[fd00:1122:3344:101::b]:47104","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":74,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:40.90644913Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:43902"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:40.906672426Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"09ab70ea-1daf-410c-b66e-d772eaa9edd4","remote_addr":"[fd00:1122:3344:101::b]:43902","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":155,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:40.912811692Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"af0d09cb-4c28-430f-8a48-6c8fe58e3b61","remote_addr":"[fd00:1122:3344:101::b]:47104","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":174,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:40.980481298Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:33156"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:40.98057112Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:41.01911399Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:41.076395286Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"55c9e636-717f-40a7-9c15-6687850b97c0","remote_addr":"[fd00:1122:3344:101::b]:33156","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":95862,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:31:41.076645593Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"58422ff7-9488-4379-ba77-78dedbc896cd","remote_addr":"[fd00:1122:3344:101::b]:33156","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":105,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:00.757490615Z","hostname":"sock","pid":1566,"uri":"/ce66c06a-7d54-47a6-860d-59da68080fee","method":"GET","req_id":"497c95ca-cb48-49ca-b7ac-6ddb1112feba","remote_addr":"[fd00:1122:3344:101::d]:56294","local_addr":"[fd00:1122:3344:101::1]:63949","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7575,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:09.134725289Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:49091"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:09.134870143Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"2665039d-9081-418b-9664-c45faedeea38","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:49091","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":75,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:10.031130878Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:35452"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:10.031248761Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"67f11904-3779-4be9-8029-542cf57685fa","remote_addr":"[fd00:1122:3344:101::a]:35452","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":69,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:10.094654442Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:10.094712553Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"204daf7c-70e2-43ff-ae19-17d347502ea0","remote_addr":"[fd00:1122:3344:101::a]:35452","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":125,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:10.1031128Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"cf248d96-fd9f-42f0-a86b-5f3504e8345d","remote_addr":"[fd00:1122:3344:101::a]:35452","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":247,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:10.175348309Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:64550"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:10.175480842Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:10.218181695Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:10.276702884Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"85179a68-af21-4098-a423-b368227b5df1","remote_addr":"[fd00:1122:3344:101::a]:64550","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":101271,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:10.276819927Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:37109"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:10.276993622Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"feb496b5-ab64-4e9f-a0b5-26baed201f38","remote_addr":"[fd00:1122:3344:101::a]:37109","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":112,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:10.277155656Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"831ee04a-41c1-4ba6-9fcb-5cf6cf2638d8","remote_addr":"[fd00:1122:3344:101::a]:64550","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":99,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:10.340400183Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:43877"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:10.34066107Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:767","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: b15378a9-5120-477b-abef-a170753103c1, kind: Service { id: 7468547b-b2f3-4993-afd8-72e9164a4301 }, name: Name(\"external-dns-7468547b-b2f3-4993-afd8-72e9164a4301\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 124])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: beea2870-fa6b-4d58-b0eb-af5a33149d11, kind: Service { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a }, name: Name(\"external-dns-8b302e6b-b09c-4ef2-8768-1e5fae50d31a\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 41])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 22195cd0-81ca-4c14-8294-e0bb8767fa51, kind: Service { id: 743f2600-b4a7-4963-8bef-692428b6800a }, name: Name(\"nexus-743f2600-b4a7-4963-8bef-692428b6800a\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 167, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 357305ec-a785-4a6e-96cf-879dc3168076, kind: Service { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f }, name: Name(\"nexus-a1e38c68-2155-4ce9-b70a-66d434b3f24f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 213])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 56f43fd0-1c39-449d-9089-5f10b7acf59b, kind: Service { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d }, name: Name(\"nexus-b4bce764-527e-4a0d-9efa-46f5760fbb1d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 31])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:10.340709411Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:10.340736882Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:10.340754913Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:10.340813424Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:10.340838965Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:10.340862625Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:10.340885776Z","hostname":"sock","pid":1566,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"13dda1fd-44bd-4f6c-af88-191a2d7e92a5","remote_addr":"[fd00:1122:3344:101::a]:43877","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":413,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:10.764734643Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:60698"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:10.764820236Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"84d4e812-8552-4d04-9555-e9a02e72ce73","remote_addr":"[fd00:1122:3344:101::c]:60698","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":47,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:10.835727349Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:10.83575245Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"ca405923-b1ca-4680-884d-72b67fe2fd0f","remote_addr":"[fd00:1122:3344:101::c]:60698","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":70,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:10.845277857Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"82f13f08-bb43-4ed3-b024-f82031235fb8","remote_addr":"[fd00:1122:3344:101::c]:60698","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":184,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:10.921691619Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:60193"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:10.921878334Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"6dd48cd9-5017-4aa1-9179-58e8ba3824d4","remote_addr":"[fd00:1122:3344:101::c]:60193","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":118,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:10.945949693Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:37483"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:10.946036406Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:10.986235651Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:11.047413851Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"6b5ce396-0802-4bf4-8bc9-8e860a329492","remote_addr":"[fd00:1122:3344:101::c]:37483","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":101413,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:11.047550185Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:65126"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:11.047589956Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:49302"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:11.047921725Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"fbde6deb-0d6f-4e67-bf2c-20ecc07c1315","remote_addr":"[fd00:1122:3344:101::b]:49302","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":241,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:11.047944146Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"349271b6-3a48-40db-8fef-2b9f59d30169","remote_addr":"[fd00:1122:3344:101::b]:65126","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":139,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:11.048022028Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"8e12c7d8-8df7-4ffa-bfd4-55dd358b74b4","remote_addr":"[fd00:1122:3344:101::c]:37483","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":266,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:11.091554402Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:11.091601844Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"4468fb9f-7df3-43a9-a3e4-a2f77d506cef","remote_addr":"[fd00:1122:3344:101::b]:65126","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":113,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:11.100895125Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"8ea30964-8c65-4e34-80ab-6783e1e82745","remote_addr":"[fd00:1122:3344:101::b]:65126","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":236,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:11.131243984Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:41270"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:11.131348356Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:11.17226888Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:11.234288473Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"6ebe13fa-8542-4662-a5a2-4fa9a01824c9","remote_addr":"[fd00:1122:3344:101::b]:41270","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":102979,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:11.234394876Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:35256"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:11.234616782Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:767","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: b15378a9-5120-477b-abef-a170753103c1, kind: Service { id: 7468547b-b2f3-4993-afd8-72e9164a4301 }, name: Name(\"external-dns-7468547b-b2f3-4993-afd8-72e9164a4301\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 124])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: beea2870-fa6b-4d58-b0eb-af5a33149d11, kind: Service { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a }, name: Name(\"external-dns-8b302e6b-b09c-4ef2-8768-1e5fae50d31a\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 41])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 22195cd0-81ca-4c14-8294-e0bb8767fa51, kind: Service { id: 743f2600-b4a7-4963-8bef-692428b6800a }, name: Name(\"nexus-743f2600-b4a7-4963-8bef-692428b6800a\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 167, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 357305ec-a785-4a6e-96cf-879dc3168076, kind: Service { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f }, name: Name(\"nexus-a1e38c68-2155-4ce9-b70a-66d434b3f24f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 213])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 56f43fd0-1c39-449d-9089-5f10b7acf59b, kind: Service { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d }, name: Name(\"nexus-b4bce764-527e-4a0d-9efa-46f5760fbb1d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 31])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:11.234660474Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:11.234688344Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:11.234698495Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:11.234722565Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:11.234736226Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:11.234753016Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:11.234767906Z","hostname":"sock","pid":1566,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"23700c0e-6333-4922-8d39-1b1734d563be","remote_addr":"[fd00:1122:3344:101::c]:35256","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":298,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:11.234922461Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"d215ed31-2128-4855-b3c4-0bb4dcc3c0b9","remote_addr":"[fd00:1122:3344:101::b]:41270","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":93,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:11.386549952Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:40557"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:11.386790589Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:767","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: b15378a9-5120-477b-abef-a170753103c1, kind: Service { id: 7468547b-b2f3-4993-afd8-72e9164a4301 }, name: Name(\"external-dns-7468547b-b2f3-4993-afd8-72e9164a4301\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 124])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: beea2870-fa6b-4d58-b0eb-af5a33149d11, kind: Service { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a }, name: Name(\"external-dns-8b302e6b-b09c-4ef2-8768-1e5fae50d31a\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 41])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 22195cd0-81ca-4c14-8294-e0bb8767fa51, kind: Service { id: 743f2600-b4a7-4963-8bef-692428b6800a }, name: Name(\"nexus-743f2600-b4a7-4963-8bef-692428b6800a\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 167, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 357305ec-a785-4a6e-96cf-879dc3168076, kind: Service { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f }, name: Name(\"nexus-a1e38c68-2155-4ce9-b70a-66d434b3f24f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 213])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 56f43fd0-1c39-449d-9089-5f10b7acf59b, kind: Service { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d }, name: Name(\"nexus-b4bce764-527e-4a0d-9efa-46f5760fbb1d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 31])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:11.38682981Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:11.386865191Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:11.386884531Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:11.386898802Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:11.386926002Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:11.386946893Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:11.386971334Z","hostname":"sock","pid":1566,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"118e7045-f440-41e6-b5f4-828bd44f7a95","remote_addr":"[fd00:1122:3344:101::b]:40557","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":354,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:30.754757509Z","hostname":"sock","pid":1566,"uri":"/ce66c06a-7d54-47a6-860d-59da68080fee","method":"GET","req_id":"e8d213f5-09bf-4564-b3ed-785d14e75122","remote_addr":"[fd00:1122:3344:101::d]:56294","local_addr":"[fd00:1122:3344:101::1]:63949","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":5353,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:39.167666454Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:38642"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:39.167802337Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"3682ba31-d6e4-4e16-8070-c64a8ddfb102","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:38642","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":73,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:39.835731191Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:48144"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:39.835890376Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"78149952-ab01-43d8-b554-294384071b28","remote_addr":"[fd00:1122:3344:101::a]:48144","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":100,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:39.850646484Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:39.850662004Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"3a7ff9b9-6e18-4545-acf3-8c3ed56e133e","remote_addr":"[fd00:1122:3344:101::a]:48144","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":71,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:39.860685805Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"a135b77a-9f5f-4616-82d4-792f4da34727","remote_addr":"[fd00:1122:3344:101::a]:48144","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":242,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:39.92867366Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:64958"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:39.928783922Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:39.966062558Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:40.026029407Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"9502ea11-f6f3-4362-8021-c8f9bca0bc28","remote_addr":"[fd00:1122:3344:101::a]:64958","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":97288,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:40.02616418Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:33506"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:40.026474168Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"aa6e35c0-573f-4cf7-a6e1-7c3fc51cd6e2","remote_addr":"[fd00:1122:3344:101::a]:33506","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":212,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:40.026568361Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"ff2acba8-4fd9-4b0c-9d73-aa3497978591","remote_addr":"[fd00:1122:3344:101::a]:64958","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":230,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:40.594737513Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:40938"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:40.594853225Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"1b5229ee-798d-444e-a8b9-8543a47080c1","remote_addr":"[fd00:1122:3344:101::c]:40938","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":42,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:40.639444289Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:40.639471329Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"d88f67b7-6d98-48fb-a776-5c8b7cdf7256","remote_addr":"[fd00:1122:3344:101::c]:40938","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":77,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:40.647310231Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"0808d17b-3ce7-45bb-8727-f1457629fde3","remote_addr":"[fd00:1122:3344:101::c]:40938","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":266,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:40.688734809Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:41644"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:40.688836732Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:40.731890094Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:40.809674013Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"f9a801f9-2626-4333-8855-b82150b01cbf","remote_addr":"[fd00:1122:3344:101::c]:41644","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":120869,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:40.809880338Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:48728"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:40.809979311Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:53154"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:40.810235758Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"8c33785f-1afc-45d0-88ca-7c8b32e85950","remote_addr":"[fd00:1122:3344:101::b]:48728","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":136,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:40.810414662Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"e3412531-2e0d-4984-8d82-86fe58d8ccfa","remote_addr":"[fd00:1122:3344:101::c]:53154","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":311,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:40.810427713Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"5cca4725-e2f2-441f-b11f-7b057090c1e0","remote_addr":"[fd00:1122:3344:101::c]:41644","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":385,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:40.853580047Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:40.853628679Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"606c6a26-fc81-4263-8cb9-ee173ddb2d14","remote_addr":"[fd00:1122:3344:101::b]:48728","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":97,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:40.86257342Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"c114055c-df0f-4c91-a5b5-33248a61779e","remote_addr":"[fd00:1122:3344:101::b]:48728","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":240,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:40.889904627Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:46587"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:40.890065462Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"baa468a3-3fcf-457e-814d-4e0511347d7d","remote_addr":"[fd00:1122:3344:101::b]:46587","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":116,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:40.89184013Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:39767"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:40.891940212Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:40.932096586Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:40.988390735Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"3b8a59f8-ef80-4eab-8a83-cdc76c9f65ae","remote_addr":"[fd00:1122:3344:101::b]:39767","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":96488,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:32:40.988646002Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"9d061a47-044a-4071-96af-53f8f4573925","remote_addr":"[fd00:1122:3344:101::b]:39767","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":105,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:00.755199277Z","hostname":"sock","pid":1566,"uri":"/ce66c06a-7d54-47a6-860d-59da68080fee","method":"GET","req_id":"5a863825-a622-40c3-81ca-2413a968f1d1","remote_addr":"[fd00:1122:3344:101::d]:56294","local_addr":"[fd00:1122:3344:101::1]:63949","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":5951,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:09.200802535Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:47817"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:09.200920948Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"51825b4f-31a1-496f-afb5-641f8a899a7c","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:47817","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":57,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:09.997481433Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:40079"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:09.997634917Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"fe9e6789-cd25-4cc2-b7d7-db165c791bf3","remote_addr":"[fd00:1122:3344:101::a]:40079","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":89,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:10.038983353Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:10.039002363Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"53625bd2-9d18-47c7-8473-b03bc258813b","remote_addr":"[fd00:1122:3344:101::a]:40079","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":84,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:10.064269845Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:50810"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:10.064602684Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"0a9fa850-c917-4ddf-a581-80df6a732fac","remote_addr":"[fd00:1122:3344:101::a]:50810","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":276,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:10.067674557Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"8401fa0f-6a20-48cd-a446-f66208246d00","remote_addr":"[fd00:1122:3344:101::a]:40079","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":220,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:10.163919984Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:56485"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:10.164031447Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:10.203360318Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:10.255190497Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"2f7d79ac-2aba-458f-8186-c0b6ad5038c7","remote_addr":"[fd00:1122:3344:101::a]:56485","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":91208,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:10.255447884Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"ba4d14b6-2b98-4a83-87ff-aab57f103800","remote_addr":"[fd00:1122:3344:101::a]:56485","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":112,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:10.697584475Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:59383"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:10.697686027Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"ee780ac4-de64-4155-ad3c-f7d0b97db57f","remote_addr":"[fd00:1122:3344:101::c]:59383","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":42,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:10.752331402Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:10.752356282Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"2ea302f6-30ac-4bdb-bc3d-692d1b328f7f","remote_addr":"[fd00:1122:3344:101::c]:59383","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":67,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:10.926879042Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:50648"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:10.927051796Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"0a467c20-2bb6-4af1-9723-d01601504f86","remote_addr":"[fd00:1122:3344:101::c]:50648","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":106,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:10.930246663Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"a903e7e7-6e38-4d13-9b97-f2992836173c","remote_addr":"[fd00:1122:3344:101::c]:59383","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":123,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:10.933745447Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:39125"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:10.93384677Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"f25931ff-e372-48ee-a058-65c5017fb3fc","remote_addr":"[fd00:1122:3344:101::b]:39125","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":32,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:10.943232883Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:46014"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:10.943360467Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"72c4913f-adb1-4256-b07e-bd7325250d84","remote_addr":"[fd00:1122:3344:101::b]:46014","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":65,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:10.975234317Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:10.975246747Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"0cba6aa0-0aa2-41a7-8acb-185ee3470938","remote_addr":"[fd00:1122:3344:101::b]:39125","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":39,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:10.984304841Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"c0e0b67f-e24c-4a15-95dc-09864a836135","remote_addr":"[fd00:1122:3344:101::b]:39125","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":104,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:10.992532403Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:41050"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:10.992641706Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:11.034370383Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:11.0891331Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"b4a5451c-f700-4d70-889b-873b1c7ad70d","remote_addr":"[fd00:1122:3344:101::c]:41050","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":96511,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:11.089237963Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:46392"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:11.089408668Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:11.12582076Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:11.178638035Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"a49be6ff-1421-48b9-aa1c-1bab47942cd9","remote_addr":"[fd00:1122:3344:101::b]:46392","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":89301,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:11.178980024Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"9cec986d-910a-4630-b853-50369a0bf977","remote_addr":"[fd00:1122:3344:101::c]:41050","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":158,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:11.178997475Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"04742362-0eb1-43c5-bca2-4884ac7f91d3","remote_addr":"[fd00:1122:3344:101::b]:46392","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":119,"response_code":"200"} {"msg":"running automatic periodic zone bundle cleanup","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:12.828477485Z","hostname":"sock","pid":1566,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:197"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:12.828560407Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:12.859546563Z","hostname":"sock","pid":1566,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:186","time_to_next_cleanup":"599.99999638s","next_cleanup":"Instant { tv_sec: 4279, tv_nsec: 694685878 }"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:26.960029214Z","hostname":"sock","pid":1566,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:30.755036729Z","hostname":"sock","pid":1566,"uri":"/ce66c06a-7d54-47a6-860d-59da68080fee","method":"GET","req_id":"9ba4c888-2907-4d6a-a5ae-885814166e0d","remote_addr":"[fd00:1122:3344:101::d]:56294","local_addr":"[fd00:1122:3344:101::1]:63949","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":5306,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:39.23300508Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34722"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:39.23374311Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"adc9c65d-e0ce-482e-bea3-59348bdfb6d1","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34722","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":72,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:39.82185596Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:42049"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:39.821961023Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"a99614ec-5bb2-4fcd-b981-681bb2ff79fe","remote_addr":"[fd00:1122:3344:101::a]:42049","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":61,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:39.861050058Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:39.861068478Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"338fd924-b0c0-441d-a445-bdd3062b0670","remote_addr":"[fd00:1122:3344:101::a]:42049","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":72,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:39.870266217Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"7f142472-2716-4710-8165-1661316b93c1","remote_addr":"[fd00:1122:3344:101::a]:42049","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":192,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:39.968433466Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:35635"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:39.9685934Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"8ccfdf4a-f457-4804-9095-c5286bad9824","remote_addr":"[fd00:1122:3344:101::a]:35635","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":112,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:40.036902473Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:59230"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:40.036998926Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:40.07570663Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:40.129333667Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"f0ed5521-b593-4b95-ac0f-5dd73734ff01","remote_addr":"[fd00:1122:3344:101::a]:59230","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":92365,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:40.129665796Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"d3416bec-6dfd-434d-821d-b2805421a059","remote_addr":"[fd00:1122:3344:101::a]:59230","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":107,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:40.77685543Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:58708"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:40.776972903Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"c4599114-01f6-433d-a166-69261e9b2209","remote_addr":"[fd00:1122:3344:101::c]:58708","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":43,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:40.836580892Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:40.836604522Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"5f72af04-d9ca-4459-bf86-3df99b4cf4a8","remote_addr":"[fd00:1122:3344:101::c]:58708","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":71,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:40.845916074Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"63e22396-5a5c-401e-8fff-14650fc0ab91","remote_addr":"[fd00:1122:3344:101::c]:58708","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":191,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:40.865248235Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:59037"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:40.865332308Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:40.903911979Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:40.960492625Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"79ab907b-589f-4d03-b4cb-3f4d06fe9555","remote_addr":"[fd00:1122:3344:101::c]:59037","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":95191,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:40.96066021Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:52282"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:40.960738622Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:35379"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:40.960792984Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:57735"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:40.960831825Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:63684"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:40.960843765Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"0bd7fa24-ebdc-4ba8-b4c6-70858d1ab44a","remote_addr":"[fd00:1122:3344:101::c]:59037","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":128,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:40.961150993Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"98355f6a-e71e-4227-9188-011573f065d9","remote_addr":"[fd00:1122:3344:101::b]:52282","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":180,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:40.961181114Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"c5f832cd-55c4-4c4e-9c70-a536dfb37ae1","remote_addr":"[fd00:1122:3344:101::b]:63684","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":250,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:40.961214795Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:40.965142561Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"2287d2bb-a2af-4ba3-a4c3-7948256aba28","remote_addr":"[fd00:1122:3344:101::c]:57735","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":3965,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:40.9991828Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:41.002992023Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:41.003048674Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"7ac67c6c-66be-478f-8321-25afdef67b33","remote_addr":"[fd00:1122:3344:101::b]:52282","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":251,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:41.015219982Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"fe6cf71a-c84c-4a11-abc6-775ad26f62ba","remote_addr":"[fd00:1122:3344:101::b]:52282","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":914,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:41.055323835Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"17d4a8bf-54ee-41d2-a76b-f1669908ba12","remote_addr":"[fd00:1122:3344:101::b]:35379","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":94284,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:33:41.055584222Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"ae627ff5-996e-4df2-9906-6763631f4168","remote_addr":"[fd00:1122:3344:101::b]:35379","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":113,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:00.756239988Z","hostname":"sock","pid":1566,"uri":"/ce66c06a-7d54-47a6-860d-59da68080fee","method":"GET","req_id":"af9044b9-e894-4406-94b8-478565ca4671","remote_addr":"[fd00:1122:3344:101::d]:56294","local_addr":"[fd00:1122:3344:101::1]:63949","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":6363,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:09.266309136Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:65414"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:09.266537002Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"3d327c8c-90bd-483a-b88e-bb3a8c8d41b9","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:65414","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":120,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:09.944729043Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:43967"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:09.944856976Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"59115aab-6768-4f0d-a0ae-ceb610eff042","remote_addr":"[fd00:1122:3344:101::a]:43967","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":89,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:09.959946173Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:09.959966454Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"e1b29101-047c-49f3-9d68-1f4975a8b138","remote_addr":"[fd00:1122:3344:101::a]:43967","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":69,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:10.095621874Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:38744"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:10.095850891Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"c27bb8ea-9324-4404-93d0-eaf7a905dd00","remote_addr":"[fd00:1122:3344:101::a]:38744","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":166,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:10.099627493Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"46c0667f-16ea-416d-a83f-b26f72299b03","remote_addr":"[fd00:1122:3344:101::a]:43967","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":220,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:10.14436156Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:37515"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:10.144457022Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:10.183444394Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:10.242195779Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"9ec09fea-7e28-4b4f-991a-0264c9a7f0f6","remote_addr":"[fd00:1122:3344:101::a]:37515","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":97776,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:10.242485287Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"61793891-57f0-4919-8e3c-811d1f2624cf","remote_addr":"[fd00:1122:3344:101::a]:37515","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":97,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:10.815667184Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:40524"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:10.81587819Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"01ac8bbb-b01b-4281-b1c8-b481993a840f","remote_addr":"[fd00:1122:3344:101::c]:40524","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":132,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:10.822234571Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:33803"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:10.822312874Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"0ee050ed-ffe3-4cd8-9f64-3db6aca67f58","remote_addr":"[fd00:1122:3344:101::b]:33803","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":30,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:10.866499626Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:10.866539997Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"056fe278-3bdd-4afa-adf7-d5fe88d749c2","remote_addr":"[fd00:1122:3344:101::b]:33803","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":74,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:10.913319799Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:62690"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:10.913404592Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"b657bdbb-ceaf-494f-9ad2-20fe4613ef69","remote_addr":"[fd00:1122:3344:101::c]:62690","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":39,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:10.932123817Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:43141"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:10.932320712Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"9a2d9140-e05e-4a66-97c9-542b9f91b2f4","remote_addr":"[fd00:1122:3344:101::b]:43141","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":109,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:10.938675114Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"b3df9a02-e047-41ed-874c-5b403cc4ea11","remote_addr":"[fd00:1122:3344:101::b]:33803","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":132,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:10.955063526Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:10.955078576Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"0db55d7d-4e3e-4131-87f8-89f0fcb7cb8e","remote_addr":"[fd00:1122:3344:101::c]:62690","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":42,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:10.964223023Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"80340bbb-675c-4de9-95c0-b236c4005b58","remote_addr":"[fd00:1122:3344:101::c]:62690","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":121,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:10.999524925Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:36555"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:10.999597587Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:11.040167792Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:11.097384686Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"466f51d1-dd47-4ccc-b32c-0b6e04d9bec4","remote_addr":"[fd00:1122:3344:101::c]:36555","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":97815,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:11.09751374Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:41110"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:11.097671664Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:11.132537455Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:11.188668359Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"0360f420-1b3b-4ca2-a35f-af05578942ac","remote_addr":"[fd00:1122:3344:101::b]:41110","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":91056,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:11.188871914Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"1c707096-75d6-440b-914b-db963cc3bc87","remote_addr":"[fd00:1122:3344:101::c]:36555","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":99,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:11.189143021Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"6351e04d-7dde-4971-bf78-f7bf8657086d","remote_addr":"[fd00:1122:3344:101::b]:41110","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":167,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:30.755397562Z","hostname":"sock","pid":1566,"uri":"/ce66c06a-7d54-47a6-860d-59da68080fee","method":"GET","req_id":"3cc7d301-6bc2-4200-a9d1-3b5a681ea14b","remote_addr":"[fd00:1122:3344:101::d]:56294","local_addr":"[fd00:1122:3344:101::1]:63949","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":6130,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:39.299963401Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:62003"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:39.300112154Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"7731f495-58ec-4128-994c-b9d6c6c3e275","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:62003","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":77,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:39.753968962Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:54939"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:39.754133656Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"0e56ed58-9f46-4c0c-b47f-76254d2ab4b0","remote_addr":"[fd00:1122:3344:101::a]:54939","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":100,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:39.78651899Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:39.786547961Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"390fe812-9089-4f26-86ab-18efd8a0c00c","remote_addr":"[fd00:1122:3344:101::a]:54939","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":96,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:39.795315877Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"6a1def41-a05f-4a77-86e5-c37b32e82005","remote_addr":"[fd00:1122:3344:101::a]:54939","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":250,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:39.912633223Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:36050"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:39.912831528Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"f993e313-cab6-4556-ba7b-ca368554cbb7","remote_addr":"[fd00:1122:3344:101::a]:36050","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":123,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:40.012525359Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:51767"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:40.012631301Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:40.050092972Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:40.100431511Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"e1108b6a-d00d-4ead-a5a5-3abdb953faae","remote_addr":"[fd00:1122:3344:101::a]:51767","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":87857,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:40.10077626Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"47990b6c-6d24-4f41-8db5-2ca471905b73","remote_addr":"[fd00:1122:3344:101::a]:51767","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":123,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:40.707048639Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:58903"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:40.707168133Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"9d5beff6-806f-46f3-8e7f-a0e94a2f7f60","remote_addr":"[fd00:1122:3344:101::b]:58903","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":53,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:40.717594014Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:51436"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:40.717685306Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"b0b8a846-9fb5-4c7f-a993-9b891d53a659","remote_addr":"[fd00:1122:3344:101::c]:51436","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":21,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:40.722324532Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:34939"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:40.722544948Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"bf12afc9-c404-4f73-8a8c-e9d980e7e850","remote_addr":"[fd00:1122:3344:101::c]:34939","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":135,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:40.734076819Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:40.734096429Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"8c2d060e-5e33-41d9-9ad6-474f5194bfb8","remote_addr":"[fd00:1122:3344:101::b]:58903","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":63,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:40.746418352Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"3384aa06-05d0-443e-8843-7e7171dad998","remote_addr":"[fd00:1122:3344:101::b]:58903","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":209,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:40.781710464Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:40.781731645Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"1f329f58-71cb-4afe-ae29-0f97dc3a4ab9","remote_addr":"[fd00:1122:3344:101::c]:51436","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":37,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:40.793733659Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"a5a0c0a6-9384-439e-ae6a-6e0da539afb9","remote_addr":"[fd00:1122:3344:101::c]:51436","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":179,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:40.797713916Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:63498"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:40.797799799Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:40.858335042Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:40.936540392Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"d18f4f6b-ce7f-46e1-9e0f-7563a288f2b9","remote_addr":"[fd00:1122:3344:101::b]:63498","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":138771,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:40.936797489Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:44200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:40.937118758Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"00a67b2a-60cb-48b8-b09a-754b00cc8817","remote_addr":"[fd00:1122:3344:101::b]:63498","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":301,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:40.93718198Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:40.950231092Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:37046"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:40.950471808Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"0ea7e2f6-98b3-412a-8d70-06e89e8a3bbf","remote_addr":"[fd00:1122:3344:101::b]:37046","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":156,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:40.975468563Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:41.028095903Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"a8e74a1f-df5d-4003-bab7-8651daa83ab9","remote_addr":"[fd00:1122:3344:101::c]:44200","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":91064,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:34:41.028704549Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"89a253a5-2d15-45bf-9340-eb8142cbbfe5","remote_addr":"[fd00:1122:3344:101::c]:44200","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":141,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:00.756014296Z","hostname":"sock","pid":1566,"uri":"/ce66c06a-7d54-47a6-860d-59da68080fee","method":"GET","req_id":"8ad380a8-5943-4b87-ab46-31b41ac5fb3d","remote_addr":"[fd00:1122:3344:101::d]:56294","local_addr":"[fd00:1122:3344:101::1]:63949","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":6886,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:09.332064243Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:58580"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:09.332193167Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"258d8f39-e8a2-4722-88d5-6f55653b45a8","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:58580","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":67,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:09.862454196Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:53274"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:09.862653371Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"9c2cb758-bdb7-466e-a1e3-07e3854ae172","remote_addr":"[fd00:1122:3344:101::a]:53274","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":146,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:10.076285816Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:59468"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:10.076420309Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"86a3bea6-e83f-48c0-b5e5-969b11af4920","remote_addr":"[fd00:1122:3344:101::a]:59468","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":68,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:10.121892936Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:10.121928577Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"d09dad2f-6fef-4277-8f09-eb1d785efcb5","remote_addr":"[fd00:1122:3344:101::a]:59468","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":108,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:10.130878709Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"2853291a-24fc-4588-b14a-c65e7175bfec","remote_addr":"[fd00:1122:3344:101::a]:59468","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":242,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:10.189555712Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:65156"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:10.189645834Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:10.227460395Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:10.285022758Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"7d666003-5f3d-47f7-9e43-a18461cff79b","remote_addr":"[fd00:1122:3344:101::a]:65156","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":95413,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:10.285278415Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"54bfa2bd-dd0a-4eab-aa7d-f294f5542f99","remote_addr":"[fd00:1122:3344:101::a]:65156","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":95,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:10.67198642Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:56339"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:10.672082663Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"3847dc51-9dee-4c61-bb9e-97a3384d948a","remote_addr":"[fd00:1122:3344:101::b]:56339","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":51,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:10.698821164Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:34181"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:10.698949018Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"d508ca90-085b-4f53-81d3-dca0ea26455d","remote_addr":"[fd00:1122:3344:101::c]:34181","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":50,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:10.744274851Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:10.744304582Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"7e57a3e0-5e39-4b76-9e25-d2fb8b55f60d","remote_addr":"[fd00:1122:3344:101::b]:56339","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":92,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:10.755068932Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"b598d6e3-6320-4213-943c-a7c0d26d56d2","remote_addr":"[fd00:1122:3344:101::b]:56339","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":205,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:10.762975295Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:10.762988476Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"2d8fecef-3c7a-41ed-84fa-26f578e285a2","remote_addr":"[fd00:1122:3344:101::c]:34181","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":43,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:10.810870898Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:64372"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:10.811039222Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"5447efa6-8445-48c6-9443-b4889bd43e31","remote_addr":"[fd00:1122:3344:101::c]:64372","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":108,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:10.814119955Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"b8d797cd-668c-421b-9b6d-4f5f6f82f3c0","remote_addr":"[fd00:1122:3344:101::c]:34181","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":131,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:10.940182647Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:60094"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:10.94027437Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:10.978337517Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:11.04404174Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"5924a59b-ea46-4929-99dc-3230e260ee61","remote_addr":"[fd00:1122:3344:101::c]:60094","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":103802,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:11.044289797Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:44350"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:11.044375849Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:40953"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:11.044607315Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"cf2a646b-6267-4b71-af9f-a49fa77194de","remote_addr":"[fd00:1122:3344:101::c]:60094","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":237,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:11.044654487Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:11.044755889Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"24895966-f51e-48bd-aa2b-4f1e79f915bf","remote_addr":"[fd00:1122:3344:101::b]:44350","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":240,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:11.085765396Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:11.144510921Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"da38f92a-7ac7-4cc3-9890-0ea6085e17ac","remote_addr":"[fd00:1122:3344:101::b]:40953","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":99981,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:11.144940433Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"df29e2a1-7650-4280-86ef-51559b00b0a5","remote_addr":"[fd00:1122:3344:101::b]:40953","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":173,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:30.754236214Z","hostname":"sock","pid":1566,"uri":"/ce66c06a-7d54-47a6-860d-59da68080fee","method":"GET","req_id":"fdf2cee1-efcf-4d06-a5da-b67f86651a6e","remote_addr":"[fd00:1122:3344:101::d]:56294","local_addr":"[fd00:1122:3344:101::1]:63949","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":6314,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:39.364114625Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:33793"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:39.364239078Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"70fa33d0-7c82-49e8-bb7b-4d7ca96b60ac","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:33793","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":71,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:39.845999078Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:46363"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:39.846147592Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"b599dc79-6e8b-4fa0-9538-410c8b24b3fe","remote_addr":"[fd00:1122:3344:101::a]:46363","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":94,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:39.894869747Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:39.894909278Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"c5baa9df-c442-41a4-975a-75a0555a6a0e","remote_addr":"[fd00:1122:3344:101::a]:46363","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":87,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:39.931017832Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:47787"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:39.931211828Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"91a064f8-8b35-4bf5-8427-327756d5833e","remote_addr":"[fd00:1122:3344:101::a]:47787","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":138,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:39.937011724Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"08ff98ca-50e3-4f6c-9d50-9d9315e203c5","remote_addr":"[fd00:1122:3344:101::a]:46363","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":225,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:39.945918414Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:56485"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:39.946015347Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:39.984578448Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:40.042353447Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"6da035cf-f690-452c-86dd-f6aed49709b4","remote_addr":"[fd00:1122:3344:101::a]:56485","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":96372,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:40.042663595Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"efa00d2e-ae7f-4ac1-b9d4-1d463bd98d64","remote_addr":"[fd00:1122:3344:101::a]:56485","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":116,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:40.601729891Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:61447"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:40.601848764Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"426aa559-1d26-45a8-9469-3662e1c821a5","remote_addr":"[fd00:1122:3344:101::c]:61447","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":40,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:40.659200012Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:40.659237873Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"d184c006-c74a-40fa-bff1-72a73e9f1d6e","remote_addr":"[fd00:1122:3344:101::c]:61447","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":111,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:40.71620462Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:61095"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:40.716326934Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"8c01174b-0f0a-468c-b48b-7310029f34d9","remote_addr":"[fd00:1122:3344:101::b]:61095","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":37,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:40.761953455Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:40.761977795Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"1ccead9f-9cf9-4feb-914e-b738233843e8","remote_addr":"[fd00:1122:3344:101::b]:61095","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":58,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:40.771342808Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:55447"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:40.772947731Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"5ff3193d-6339-45b9-ad83-1257c5dbfa28","remote_addr":"[fd00:1122:3344:101::c]:55447","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":129,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:40.773146317Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"65e9ebe5-f234-4e44-95bd-ecfd43b0d66f","remote_addr":"[fd00:1122:3344:101::b]:61095","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":140,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:40.788417419Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"ec29ec17-cc05-4fe8-a4cb-fdb8fb604e60","remote_addr":"[fd00:1122:3344:101::c]:61447","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":258,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:40.790219367Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:49967"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:40.790418083Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:40.820037332Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:38935"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:40.820940146Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:40.829871237Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"b233d8c3-e5be-43fb-a16d-5db8b8a6ef53","remote_addr":"[fd00:1122:3344:101::b]:38935","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":488,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:40.85775216Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:34225"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:40.882809736Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:40.882874188Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"e2be86ed-9f00-4cd8-8cc0-a6b0bf7d4797","remote_addr":"[fd00:1122:3344:101::b]:49967","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":92536,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:40.886942287Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"0fe2949c-26f7-440f-8f87-34c04258330a","remote_addr":"[fd00:1122:3344:101::b]:49967","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":804,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:40.920999046Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:40.972896617Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"f367923d-6c91-4bcb-a61d-59c4058b926a","remote_addr":"[fd00:1122:3344:101::c]:34225","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":115012,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:35:40.973244776Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"7dabd697-27c1-41e5-b0e5-0d987bb7e55b","remote_addr":"[fd00:1122:3344:101::c]:34225","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":150,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:00.753496421Z","hostname":"sock","pid":1566,"uri":"/ce66c06a-7d54-47a6-860d-59da68080fee","method":"GET","req_id":"6ffdc921-8858-41c6-ad63-2179cc9c3bc6","remote_addr":"[fd00:1122:3344:101::d]:56294","local_addr":"[fd00:1122:3344:101::1]:63949","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":4920,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:09.395636872Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:50019"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:09.395829888Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"d5eb33bb-1512-4e23-8215-1343380b6eb6","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:50019","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":88,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:09.928056719Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:58266"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:09.928248195Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"7fb544cf-79d5-4c0e-8405-2fc504e54d1f","remote_addr":"[fd00:1122:3344:101::a]:58266","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":136,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:09.974906854Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:09.974925954Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"3e158545-c809-4618-b6a3-f49d2305ae86","remote_addr":"[fd00:1122:3344:101::a]:58266","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":84,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:09.984312228Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"d1766f49-c6e3-450c-9ebd-3adad725724b","remote_addr":"[fd00:1122:3344:101::a]:58266","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":202,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:10.012348244Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:34221"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:10.01255613Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"1bcdeaa8-04ef-467b-8682-c7446395d066","remote_addr":"[fd00:1122:3344:101::a]:34221","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":159,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:10.075067346Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:33930"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:10.07519596Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:10.114238934Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:10.176657447Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"7f3602aa-0c95-4dfb-ac2a-32bd75571b7c","remote_addr":"[fd00:1122:3344:101::a]:33930","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":101523,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:10.176994207Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"156dd31e-3fa5-46ea-bfc9-e6e61f71c468","remote_addr":"[fd00:1122:3344:101::a]:33930","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":121,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:10.725341224Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:41643"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:10.725554759Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"7f224eb6-9b5e-417d-a847-e8cdcb918665","remote_addr":"[fd00:1122:3344:101::c]:41643","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":107,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:10.778008415Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:10.778047386Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"0b771da0-0335-4584-ba5d-2e79beddc90b","remote_addr":"[fd00:1122:3344:101::c]:41643","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":79,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:10.787911392Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"0f1cf633-909d-44cb-9ea1-3733b78e8b6d","remote_addr":"[fd00:1122:3344:101::c]:41643","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":221,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:10.887320855Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:38010"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:10.887491219Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"75424a69-5bc6-439b-9eec-fd61eb2f3c40","remote_addr":"[fd00:1122:3344:101::c]:38010","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":123,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:10.899671538Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:45166"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:10.89975646Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:10.947134009Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:11.021816054Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"5e80f5d4-80f4-4ddd-82c5-8ffff6d6ed99","remote_addr":"[fd00:1122:3344:101::c]:45166","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":122085,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:11.021954148Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:40017"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:11.022081501Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"4df2fe26-ba5b-4858-9516-c0133e7fe242","remote_addr":"[fd00:1122:3344:101::b]:40017","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":54,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:11.022241585Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"5906bb36-cba5-4fc2-aed5-e5eb7d661432","remote_addr":"[fd00:1122:3344:101::c]:45166","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":101,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:11.070521518Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:11.070543569Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"119c7472-6f3f-48be-a67f-e49bca69a2e7","remote_addr":"[fd00:1122:3344:101::b]:40017","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":86,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:11.080034965Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"ca80fe26-6da6-48d2-b829-4a57af0af4bc","remote_addr":"[fd00:1122:3344:101::b]:40017","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":302,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:11.123815746Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:35116"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:11.12396437Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"8371d08f-4b62-44a7-9db2-df26c715d538","remote_addr":"[fd00:1122:3344:101::b]:35116","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":103,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:11.136246021Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:48804"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:11.136343874Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:11.17289355Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:11.229988641Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"90bc75d0-5916-4573-9cbb-a3ed577639d4","remote_addr":"[fd00:1122:3344:101::b]:48804","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":93686,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:11.23032024Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"22f7b3a7-3775-454d-9eec-40e30693f64c","remote_addr":"[fd00:1122:3344:101::b]:48804","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":121,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:30.753441346Z","hostname":"sock","pid":1566,"uri":"/ce66c06a-7d54-47a6-860d-59da68080fee","method":"GET","req_id":"b5890267-a438-4fef-b5fd-e748ffe89b08","remote_addr":"[fd00:1122:3344:101::d]:56294","local_addr":"[fd00:1122:3344:101::1]:63949","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":5681,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:39.427674533Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:40830"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:39.427816497Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"6953195c-fe07-4da5-8912-4ad0e6e4a43f","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:40830","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":85,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:39.781128861Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:60605"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:39.781326126Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"9818f8b4-c60b-4b85-b41f-475b55f730c1","remote_addr":"[fd00:1122:3344:101::a]:60605","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":148,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:39.848340565Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:49429"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:39.848464178Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"422c0951-7f99-401d-b636-c0a889228214","remote_addr":"[fd00:1122:3344:101::a]:49429","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":84,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:39.868767786Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:39.868794657Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"c9593772-223f-49e0-95c3-227ce891d311","remote_addr":"[fd00:1122:3344:101::a]:49429","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":151,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:39.877549963Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"639557d3-636a-4748-bc15-52bb2bcda546","remote_addr":"[fd00:1122:3344:101::a]:49429","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":332,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:39.929105064Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:33804"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:39.929224647Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:39.969994637Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:40.02827261Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"f835466b-9d5e-4746-af8d-53f86d294e44","remote_addr":"[fd00:1122:3344:101::a]:33804","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":99096,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:40.028580929Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"6b751832-db2d-4bc7-8c09-c0e96a34cef9","remote_addr":"[fd00:1122:3344:101::a]:33804","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":108,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:40.544390207Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:47048"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:40.544553251Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"51670c88-67d7-4130-b336-26a3f49a37b3","remote_addr":"[fd00:1122:3344:101::c]:47048","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":97,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:40.625752463Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:40.625777693Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"57ac34ae-0f92-4df5-9a2b-1afd5175d691","remote_addr":"[fd00:1122:3344:101::c]:47048","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":64,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:40.636661387Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"13a986bc-08f5-4cfc-ac01-ad3a3f237f74","remote_addr":"[fd00:1122:3344:101::c]:47048","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":210,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:40.688450524Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:47581"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:40.688617709Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"5a227ec5-91f8-4f94-a51e-218c9a7772c8","remote_addr":"[fd00:1122:3344:101::c]:47581","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":101,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:40.729304977Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:36891"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:40.729507162Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"6ed12758-8789-4cf0-9dab-d57c2b1595f5","remote_addr":"[fd00:1122:3344:101::b]:36891","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":98,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:40.750784276Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:32817"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:40.75090624Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:40.791966168Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:40.85913033Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"aa32c122-8b92-4760-87c9-c75ccf65bf39","remote_addr":"[fd00:1122:3344:101::c]:32817","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":108256,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:40.859448049Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"9037f2a8-0971-480d-bdaf-0d643c017903","remote_addr":"[fd00:1122:3344:101::c]:32817","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":138,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:40.85949544Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:40.85950244Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"514974c1-beb9-46ef-918a-337cc176a0e1","remote_addr":"[fd00:1122:3344:101::b]:36891","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":121,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:40.863870048Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:46236"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:40.864131535Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"3c511afa-e63d-4ccc-aa1c-6aca347807a4","remote_addr":"[fd00:1122:3344:101::b]:46236","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":168,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:40.868454632Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"7663a6b5-ad75-4a07-b351-51ba6bad89e5","remote_addr":"[fd00:1122:3344:101::b]:36891","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":321,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:40.877949348Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:37929"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:40.87803432Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:40.914240197Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:40.969991882Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"be0cd398-7603-4a63-8585-bf8f90eafb4c","remote_addr":"[fd00:1122:3344:101::b]:37929","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":91998,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:36:40.970274259Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"26356018-86cb-4d3c-89c9-21e25cae1f63","remote_addr":"[fd00:1122:3344:101::b]:37929","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":104,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:00.753772382Z","hostname":"sock","pid":1566,"uri":"/ce66c06a-7d54-47a6-860d-59da68080fee","method":"GET","req_id":"aae077b9-daeb-4593-9380-e14706ece2e6","remote_addr":"[fd00:1122:3344:101::d]:56294","local_addr":"[fd00:1122:3344:101::1]:63949","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":5522,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:09.460845925Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:50164"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:09.46103989Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"7b614639-0dc0-4a7b-b2e8-6946613dbb6b","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:50164","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":89,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:09.882502563Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:42039"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:09.882686468Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"1aa1739c-952a-4354-b4a4-e59e2c20280f","remote_addr":"[fd00:1122:3344:101::a]:42039","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":128,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:09.942456021Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:09.942490522Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"72fa5b2d-a441-42fb-8137-6c54e94313dd","remote_addr":"[fd00:1122:3344:101::a]:42039","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":122,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:09.95240806Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"1ebfc461-53e7-4327-91c7-d2027f75d0a2","remote_addr":"[fd00:1122:3344:101::a]:42039","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":294,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.132833038Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:55826"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.133040944Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"7006a2c5-05ec-4f97-94f9-6ec91f62e600","remote_addr":"[fd00:1122:3344:101::a]:55826","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":166,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.17106016Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:51039"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.171187943Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.212762125Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.26779467Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"e256e274-449b-4151-b00f-006a123b3af6","remote_addr":"[fd00:1122:3344:101::a]:51039","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":96654,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.267918123Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:58440"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.268123539Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.26815626Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.268320354Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"10b7bce6-78e3-4bb7-a4aa-11870ac9a499","remote_addr":"[fd00:1122:3344:101::a]:51039","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":236,"response_code":"200"} {"msg":"Reading ledger from /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.268506689Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.270808781Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.283072382Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.283143694Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_57cb94f0-4114-4bd3-970a-a0245b825f9d"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.30709109Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:40602"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.307412919Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:767","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: b15378a9-5120-477b-abef-a170753103c1, kind: Service { id: 7468547b-b2f3-4993-afd8-72e9164a4301 }, name: Name(\"external-dns-7468547b-b2f3-4993-afd8-72e9164a4301\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 124])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: beea2870-fa6b-4d58-b0eb-af5a33149d11, kind: Service { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a }, name: Name(\"external-dns-8b302e6b-b09c-4ef2-8768-1e5fae50d31a\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 41])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 22195cd0-81ca-4c14-8294-e0bb8767fa51, kind: Service { id: 743f2600-b4a7-4963-8bef-692428b6800a }, name: Name(\"nexus-743f2600-b4a7-4963-8bef-692428b6800a\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 167, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 357305ec-a785-4a6e-96cf-879dc3168076, kind: Service { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f }, name: Name(\"nexus-a1e38c68-2155-4ce9-b70a-66d434b3f24f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 213])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 56f43fd0-1c39-449d-9089-5f10b7acf59b, kind: Service { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d }, name: Name(\"nexus-b4bce764-527e-4a0d-9efa-46f5760fbb1d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 31])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.30746929Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.307511431Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.307524572Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.307577553Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.307605864Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.307637245Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.307667946Z","hostname":"sock","pid":1566,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"f8b2c824-96af-4d10-aaf4-cc71147157d6","remote_addr":"[fd00:1122:3344:101::a]:40602","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":455,"response_code":"204"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.308112478Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.308130208Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_27bb8d18-eb2e-4bb2-b35c-c031637c236f"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.330124352Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.330196904Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_87e49e9a-4ab3-4574-b920-463506a8978e"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.351368845Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.351422147Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_b471fd39-9806-47fa-8ddb-0ca2f42aaf5d"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.372378872Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.372438403Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_03ea8739-b702-4a4b-b400-0745bd6b4f96"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.400356247Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.400430279Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_d8acd544-80aa-4e82-b375-eb038fe27547"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.435483995Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.435562927Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_0aaa1466-164a-4540-ae41-ab17e9fc6bc8"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.470234233Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.470329985Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_56c8079e-8d4e-4506-b6ff-2f0a5efab320"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.513702546Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.513772768Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_276e9391-34d6-4a99-bd91-776ecada771e"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.536162292Z","hostname":"sock","pid":1566,"uri":"/inventory","method":"GET","req_id":"9689b130-c723-46b1-a906-cd854b805b39","remote_addr":"[fd00:1122:3344:101::a]:58440","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":268093,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.827212295Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:34415"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.827317468Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"56d69340-89a9-41cf-8af2-d5f7ce7487db","remote_addr":"[fd00:1122:3344:101::c]:34415","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":46,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.892528998Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.892574889Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"ba30cd90-b2a9-4e07-a9b1-cabe0e80cc07","remote_addr":"[fd00:1122:3344:101::c]:34415","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":83,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.896738192Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:35461"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.896845294Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"195f9f65-e4ab-434c-b717-9af41e3f4796","remote_addr":"[fd00:1122:3344:101::b]:35461","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":27,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.909246779Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"af7d6602-b750-4577-9c55-d6999e70bb62","remote_addr":"[fd00:1122:3344:101::c]:34415","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":202,"response_code":"204"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.95746541Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.957516132Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"aedb1d0c-6e20-4bb6-8020-439a2d038ffc","remote_addr":"[fd00:1122:3344:101::b]:35461","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":113,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.973155904Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"ba5b465b-b917-4561-8edc-6a8cd170339f","remote_addr":"[fd00:1122:3344:101::b]:35461","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":251,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.985640181Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:48592"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:10.985840896Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"a903b600-2a82-42ba-8e56-347dc50bdfcb","remote_addr":"[fd00:1122:3344:101::c]:48592","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":125,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.000774859Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:38340"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.000885122Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.045242349Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.114667192Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"df755040-e2c9-4792-8a96-90211ab6ebbc","remote_addr":"[fd00:1122:3344:101::b]:38340","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":113828,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.11493799Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:57459"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.115027202Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:37453"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.115086284Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:63045"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.115277289Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.11530561Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.115364401Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"43611c32-0a9b-4809-b74e-4c2ce4170514","remote_addr":"[fd00:1122:3344:101::b]:38340","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":396,"response_code":"200"} {"msg":"Reading ledger from /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.115415423Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.115489325Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.115519726Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"639bafdb-afc4-44f1-b354-0df557e93c73","remote_addr":"[fd00:1122:3344:101::b]:63045","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":304,"response_code":"200"} {"msg":"Reading ledger from /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.121128687Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.135091394Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.135382331Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_57cb94f0-4114-4bd3-970a-a0245b825f9d"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.156727077Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.164903568Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.16497069Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_27bb8d18-eb2e-4bb2-b35c-c031637c236f"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.174162297Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:37141"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.1746402Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:767","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: b15378a9-5120-477b-abef-a170753103c1, kind: Service { id: 7468547b-b2f3-4993-afd8-72e9164a4301 }, name: Name(\"external-dns-7468547b-b2f3-4993-afd8-72e9164a4301\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 124])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: beea2870-fa6b-4d58-b0eb-af5a33149d11, kind: Service { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a }, name: Name(\"external-dns-8b302e6b-b09c-4ef2-8768-1e5fae50d31a\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 41])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 22195cd0-81ca-4c14-8294-e0bb8767fa51, kind: Service { id: 743f2600-b4a7-4963-8bef-692428b6800a }, name: Name(\"nexus-743f2600-b4a7-4963-8bef-692428b6800a\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 167, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 357305ec-a785-4a6e-96cf-879dc3168076, kind: Service { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f }, name: Name(\"nexus-a1e38c68-2155-4ce9-b70a-66d434b3f24f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 213])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 56f43fd0-1c39-449d-9089-5f10b7acf59b, kind: Service { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d }, name: Name(\"nexus-b4bce764-527e-4a0d-9efa-46f5760fbb1d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 31])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.174698712Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.174752823Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.174780524Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.174820395Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.174877107Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.174916557Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.174947348Z","hostname":"sock","pid":1566,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"bfff024d-cb3c-47e0-9307-48144f751d77","remote_addr":"[fd00:1122:3344:101::b]:37141","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":626,"response_code":"204"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.18872025Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.188804272Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_87e49e9a-4ab3-4574-b920-463506a8978e"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.211876755Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.211974188Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_b471fd39-9806-47fa-8ddb-0ca2f42aaf5d"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.219617794Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:58577"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.219892471Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:767","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: b15378a9-5120-477b-abef-a170753103c1, kind: Service { id: 7468547b-b2f3-4993-afd8-72e9164a4301 }, name: Name(\"external-dns-7468547b-b2f3-4993-afd8-72e9164a4301\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 124])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: beea2870-fa6b-4d58-b0eb-af5a33149d11, kind: Service { id: 8b302e6b-b09c-4ef2-8768-1e5fae50d31a }, name: Name(\"external-dns-8b302e6b-b09c-4ef2-8768-1e5fae50d31a\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 169, 41])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 22195cd0-81ca-4c14-8294-e0bb8767fa51, kind: Service { id: 743f2600-b4a7-4963-8bef-692428b6800a }, name: Name(\"nexus-743f2600-b4a7-4963-8bef-692428b6800a\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 167, 161])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 357305ec-a785-4a6e-96cf-879dc3168076, kind: Service { id: a1e38c68-2155-4ce9-b70a-66d434b3f24f }, name: Name(\"nexus-a1e38c68-2155-4ce9-b70a-66d434b3f24f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 214, 213])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 56f43fd0-1c39-449d-9089-5f10b7acf59b, kind: Service { id: b4bce764-527e-4a0d-9efa-46f5760fbb1d }, name: Name(\"nexus-b4bce764-527e-4a0d-9efa-46f5760fbb1d\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 200, 31])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.219931612Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.219969633Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.219987624Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.220009674Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.220030025Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.220048815Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:786","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.220073346Z","hostname":"sock","pid":1566,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"a9cc385e-aa99-45d0-a738-f9075d5bf342","remote_addr":"[fd00:1122:3344:101::c]:58577","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":356,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.221739061Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"a12f8485-9e89-4ce8-98fd-80a41342bcb9","remote_addr":"[fd00:1122:3344:101::c]:37453","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":106390,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.221958437Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"6ec9203c-071f-44de-8018-2cb86f04282d","remote_addr":"[fd00:1122:3344:101::c]:37453","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":89,"response_code":"200"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.235372669Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.23543185Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_03ea8739-b702-4a4b-b400-0745bd6b4f96"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.256371555Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.256442057Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_d8acd544-80aa-4e82-b375-eb038fe27547"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.276353615Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.276414736Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_0aaa1466-164a-4540-ae41-ab17e9fc6bc8"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.29657348Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.296634972Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_56c8079e-8d4e-4506-b6ff-2f0a5efab320"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.317544046Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.317582257Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_276e9391-34d6-4a99-bd91-776ecada771e"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:11.328176453Z","hostname":"sock","pid":1566,"uri":"/inventory","method":"GET","req_id":"caff92bf-a48e-41e6-878a-162e57d9e5d9","remote_addr":"[fd00:1122:3344:101::c]:57459","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":213028,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:14.081225982Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:64103"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:14.081417168Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:14.081429408Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Reading ledger from /pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:14.08149535Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:14.08188496Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 57cb94f0-4114-4bd3-970a-a0245b825f9d (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:14.095087787Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:14.095175989Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_57cb94f0-4114-4bd3-970a-a0245b825f9d"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 27bb8d18-eb2e-4bb2-b35c-c031637c236f (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:14.119186327Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:14.119247228Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_27bb8d18-eb2e-4bb2-b35c-c031637c236f"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 87e49e9a-4ab3-4574-b920-463506a8978e (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:14.144289834Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:14.144339456Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_87e49e9a-4ab3-4574-b920-463506a8978e"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: b471fd39-9806-47fa-8ddb-0ca2f42aaf5d (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:14.168375244Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:14.168415845Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_b471fd39-9806-47fa-8ddb-0ca2f42aaf5d"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 03ea8739-b702-4a4b-b400-0745bd6b4f96 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:14.189041462Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:14.189083083Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_03ea8739-b702-4a4b-b400-0745bd6b4f96"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: d8acd544-80aa-4e82-b375-eb038fe27547 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:14.208503077Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:14.208553558Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_d8acd544-80aa-4e82-b375-eb038fe27547"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 0aaa1466-164a-4540-ae41-ab17e9fc6bc8 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:14.230248223Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:14.230313295Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_0aaa1466-164a-4540-ae41-ab17e9fc6bc8"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 56c8079e-8d4e-4506-b6ff-2f0a5efab320 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:14.251700092Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:14.251774184Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_56c8079e-8d4e-4506-b6ff-2f0a5efab320"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 276e9391-34d6-4a99-bd91-776ecada771e (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:14.275148365Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:14.275220907Z","hostname":"sock","pid":1566,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1118","zpool":"oxp_276e9391-34d6-4a99-bd91-776ecada771e"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:14.287719084Z","hostname":"sock","pid":1566,"uri":"/inventory","method":"GET","req_id":"67660328-5733-495b-ab43-f54b0c4efa75","remote_addr":"[fd00:1122:3344:101::b]:64103","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":206370,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:30.752767381Z","hostname":"sock","pid":1566,"uri":"/ce66c06a-7d54-47a6-860d-59da68080fee","method":"GET","req_id":"d29ba10c-b56a-4d02-888c-cda31c809e99","remote_addr":"[fd00:1122:3344:101::d]:56294","local_addr":"[fd00:1122:3344:101::1]:63949","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":5318,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:39.496248897Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:57266"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:39.49637135Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"5dfec8cd-37a2-49d5-ac59-82739b1d8221","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:57266","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":61,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:39.938290535Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:41300"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:39.938441849Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"57b0710e-db74-4906-ab1d-5807c2434635","remote_addr":"[fd00:1122:3344:101::a]:41300","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":89,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:39.984380199Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:39.98441323Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"95e82014-cb23-4375-9bd5-365b59240ef7","remote_addr":"[fd00:1122:3344:101::a]:41300","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":88,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:39.994346928Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"c18e77f6-5834-4b98-b616-66f5f1f2922c","remote_addr":"[fd00:1122:3344:101::a]:41300","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":237,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:39.994839501Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:43082"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:39.994959504Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:40.042006914Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:40.099804284Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"e7b7c657-210b-4b92-b02f-fd6f9c66c294","remote_addr":"[fd00:1122:3344:101::a]:43082","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":104894,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:40.100081401Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:60537"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:40.100274516Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"0eb3df98-e2c3-4709-9087-2128627cffaf","remote_addr":"[fd00:1122:3344:101::a]:43082","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":198,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:40.10041898Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"abce60ed-2529-4ca7-b25a-ad17b328f382","remote_addr":"[fd00:1122:3344:101::a]:60537","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":198,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:40.582420046Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:61976"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:40.582476028Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"859c004f-547d-4757-8efa-ef19b16c7ad7","remote_addr":"[fd00:1122:3344:101::c]:61976","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":79,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:40.645790386Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:40.645837238Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"beb826a3-7d0c-4ace-a2c9-9b7863c5d764","remote_addr":"[fd00:1122:3344:101::c]:61976","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":65,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:40.654923333Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"dc3fe0b0-9841-4974-98ff-23b4ae5e97dd","remote_addr":"[fd00:1122:3344:101::c]:61976","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":203,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:40.714635814Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:50217"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:40.714741287Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:40.756927435Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:40.819318109Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"2a9fa8f1-5693-48c4-871a-2dd9f8fb70c7","remote_addr":"[fd00:1122:3344:101::c]:50217","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":104620,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:40.819580136Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:63232"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:40.819628497Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:53241"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:40.819705289Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:58382"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:40.820029308Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"4195b209-ab7f-4f40-8ace-d8154c162e5d","remote_addr":"[fd00:1122:3344:101::c]:50217","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":370,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:40.82010488Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"bb40abf4-ef61-4bcb-a5e1-0fb050ed6526","remote_addr":"[fd00:1122:3344:101::c]:58382","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":271,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:40.820122461Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"0ef3b2ed-5d7a-4906-96ae-e71ba71632ea","remote_addr":"[fd00:1122:3344:101::b]:63232","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":222,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:40.820141571Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"4b4f25f2-7e36-4dae-b04c-058b26fc05b7","remote_addr":"[fd00:1122:3344:101::b]:53241","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":266,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:40.867685064Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:40.867714075Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"78bb187c-7cb6-43c9-b526-caa75d9bc202","remote_addr":"[fd00:1122:3344:101::b]:63232","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":89,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:40.877122529Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"04a4d52f-22b2-4e55-a27f-55e1db205d9c","remote_addr":"[fd00:1122:3344:101::b]:63232","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":238,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:40.886158893Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:43001"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:40.886252435Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:40.925871554Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:40.986776898Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"5e78797a-c492-4892-8983-944819938a0d","remote_addr":"[fd00:1122:3344:101::b]:43001","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":100546,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:40.987083956Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"46ffffae-ba24-4dd1-bd8f-ebf52b688e33","remote_addr":"[fd00:1122:3344:101::b]:43001","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":113,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:46.545119135Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:39874"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:46.545312551Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"c3d4eb51-a740-44f3-95eb-f25f6b1ea67d","remote_addr":"[fd00:1122:3344:101::c]:39874","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":133,"response_code":"200"} {"msg":"Mapping virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:46.554880299Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:874","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(3887612) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:46.5549461Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"PUT","req_id":"76c1aae1-031a-43db-bc16-8b10535b7720","remote_addr":"[fd00:1122:3344:101::c]:39874","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7284,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:46.570966253Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:57156"} {"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:46.587093268Z","hostname":"sock","pid":1566,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:555","metadata":"InstanceMetadata { silo_id: 25a10c65-c21a-4403-8acf-3f8c60b10e2b, project_id: 13f9223c-15ec-40ed-8ceb-79dbb1e92fe0 }","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2025-01-15T02:37:46.570782978Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 8bfad461-5ff5-4bae-8dc5-5c05bfcee1cf, kind: Instance { id: 62d616e1-f148-4a1c-90a4-4e962236db27 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(3887612), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 8bfad461-5ff5-4bae-8dc5-5c05bfcee1cf, kind: Instance { id: 62d616e1-f148-4a1c-90a4-4e962236db27 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(3887612), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 8bfad461-5ff5-4bae-8dc5-5c05bfcee1cf, kind: Instance { id: 62d616e1-f148-4a1c-90a4-4e962236db27 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(3887612), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(3887612))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 8bfad461-5ff5-4bae-8dc5-5c05bfcee1cf, kind: Instance { id: 62d616e1-f148-4a1c-90a4-4e962236db27 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(3887612), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [InstanceDisk { disk_id: 9dcc8a0b-5f10-4839-b5cf-9d62d7bafdb9, slot: 0, read_only: false, vcr_json: .., name: \"disk-1818f16b6a1\" }], boot_settings: Some(InstanceBootSettings { order: [9dcc8a0b-5f10-4839-b5cf-9d62d7bafdb9] }), cloud_init_bytes: Some(..) }","migration_id":"None","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27"} {"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:46.587130849Z","hostname":"sock","pid":1566,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:588","migration_id":"None","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27"} {"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:46.58789898Z","hostname":"sock","pid":1566,"propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","component":"InstanceManager","file":"sled-agent/src/instance.rs:1563","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 8bfad461-5ff5-4bae-8dc5-5c05bfcee1cf, kind: Instance { id: 62d616e1-f148-4a1c-90a4-4e962236db27 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(3887612), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 8bfad461-5ff5-4bae-8dc5-5c05bfcee1cf, kind: Instance { id: 62d616e1-f148-4a1c-90a4-4e962236db27 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(3887612), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 8bfad461-5ff5-4bae-8dc5-5c05bfcee1cf, kind: Instance { id: 62d616e1-f148-4a1c-90a4-4e962236db27 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(3887612), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(3887612))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 8bfad461-5ff5-4bae-8dc5-5c05bfcee1cf, kind: Instance { id: 62d616e1-f148-4a1c-90a4-4e962236db27 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(3887612), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [InstanceDisk { disk_id: 9dcc8a0b-5f10-4839-b5cf-9d62d7bafdb9, slot: 0, read_only: false, vcr_json: .., name: \"disk-1818f16b6a1\" }], boot_settings: Some(InstanceBootSettings { order: [9dcc8a0b-5f10-4839-b5cf-9d62d7bafdb9] }), cloud_init_bytes: Some(..) }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2025-01-15T02:37:46.570782978Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400, migration_id: None }","migration_id":"None","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:46.591397364Z","hostname":"sock","pid":1566,"uri":"/vmms/04760650-1e85-40f1-9f70-557a11b03e88","method":"PUT","req_id":"720ebc7e-17b4-40d1-b2d0-d58637100b2e","remote_addr":"[fd00:1122:3344:101::c]:57156","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":20342,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:46.773142698Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:54924"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:46.780521747Z","hostname":"sock","pid":1566,"uri":"/vmms/04760650-1e85-40f1-9f70-557a11b03e88/state","method":"PUT","req_id":"97c93334-970f-456e-a0f0-fcf637aac8c5","remote_addr":"[fd00:1122:3344:101::c]:54924","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7299,"response_code":"200"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:46.781939896Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:447","port":"Port { inner: PortInner(PortData { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 240, 0, 0]), slot: 0, vni: Vni { inner: 3887612 }, subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 } }) }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:46.782140991Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"Configuring new Omicron zone: oxz_propolis-server_04760650-1e85-40f1-9f70-557a11b03e88","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:46.821675418Z","hostname":"sock","pid":1566,"instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","component":"InstanceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_propolis-server_04760650-1e85-40f1-9f70-557a11b03e88","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:46.84582841Z","hostname":"sock","pid":1566,"instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","component":"InstanceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:46.987715378Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:41761"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:46.987888033Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"5fb25711-f9ff-44e8-90a6-a632a9c55041","remote_addr":"[fd00:1122:3344:101::c]:41761","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":63,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:47.006338941Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {8bfad461-5ff5-4bae-8dc5-5c05bfcee1cf: {10.151.2.175: {1187a63a-5e12-4ea5-9717-553d241f7bb3}, 10.151.2.176: {1187a63a-5e12-4ea5-9717-553d241f7bb3}}} }"} {"msg":"Profile for oxz_propolis-server_04760650-1e85-40f1-9f70-557a11b03e88:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:48.61726362Z","hostname":"sock","pid":1566,"instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_propolis-server_04760650-1e85-40f1-9f70-557a11b03e88 zone","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:48.617490106Z","hostname":"sock","pid":1566,"zone":"oxz_propolis-server_04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_propolis-server_04760650-1e85-40f1-9f70-557a11b03e88\") failed: Property not found. retry in 48.653875ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:37:48.918046787Z","hostname":"sock","pid":1566,"zone":"oxz_propolis-server_04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","component":"InstanceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_propolis-server_04760650-1e85-40f1-9f70-557a11b03e88\") failed: Property not found. retry in 137.885749ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:37:48.976276848Z","hostname":"sock","pid":1566,"zone":"oxz_propolis-server_04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","component":"InstanceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_propolis-server_04760650-1e85-40f1-9f70-557a11b03e88\") failed: Property not found. retry in 173.122165ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:37:49.132148324Z","hostname":"sock","pid":1566,"zone":"oxz_propolis-server_04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","component":"InstanceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_propolis-server_04760650-1e85-40f1-9f70-557a11b03e88\") failed: Property not found. retry in 281.700667ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:37:49.321229516Z","hostname":"sock","pid":1566,"zone":"oxz_propolis-server_04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","component":"InstanceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_propolis-server_04760650-1e85-40f1-9f70-557a11b03e88\") failed: Property not found. retry in 1.079002128s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:37:49.619722921Z","hostname":"sock","pid":1566,"zone":"oxz_propolis-server_04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","component":"InstanceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_propolis-server_04760650-1e85-40f1-9f70-557a11b03e88\") failed: Property not found. retry in 515.657294ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:37:50.710051063Z","hostname":"sock","pid":1566,"zone":"oxz_propolis-server_04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","component":"InstanceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_propolis-server_04760650-1e85-40f1-9f70-557a11b03e88\") failed: Property not found. retry in 718.290964ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:37:51.238560624Z","hostname":"sock","pid":1566,"zone":"oxz_propolis-server_04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","component":"InstanceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_propolis-server_04760650-1e85-40f1-9f70-557a11b03e88\") failed: Property not found. retry in 634.689341ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:37:51.967972007Z","hostname":"sock","pid":1566,"zone":"oxz_propolis-server_04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","component":"InstanceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_propolis-server_04760650-1e85-40f1-9f70-557a11b03e88\") failed: Property not found. retry in 644.389295ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:37:52.613202937Z","hostname":"sock","pid":1566,"zone":"oxz_propolis-server_04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","component":"InstanceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_propolis-server_04760650-1e85-40f1-9f70-557a11b03e88\") failed: Property not found. retry in 1.348786089s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:37:53.269542948Z","hostname":"sock","pid":1566,"zone":"oxz_propolis-server_04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","component":"InstanceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_propolis-server_04760650-1e85-40f1-9f70-557a11b03e88\") failed: Property not found. retry in 1.068134919s","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:37:54.629850165Z","hostname":"sock","pid":1566,"zone":"oxz_propolis-server_04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","component":"InstanceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"Started propolis in zone: oxz_propolis-server_04760650-1e85-40f1-9f70-557a11b03e88","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:55.728652846Z","hostname":"sock","pid":1566,"instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","component":"InstanceManager","file":"sled-agent/src/instance.rs:2068"} {"msg":"wait for service Some(\"oxz_propolis-server_04760650-1e85-40f1-9f70-557a11b03e88\") failed: Property not found. retry in 53.923955ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:37:55.73880178Z","hostname":"sock","pid":1566,"instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","component":"InstanceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_propolis-server_04760650-1e85-40f1-9f70-557a11b03e88\") failed: Property not found. retry in 126.629673ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:37:55.803971728Z","hostname":"sock","pid":1566,"instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","component":"InstanceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"wait for service Some(\"oxz_propolis-server_04760650-1e85-40f1-9f70-557a11b03e88\") failed: Property not found. retry in 292.599169ms","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:37:55.941364106Z","hostname":"sock","pid":1566,"instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","component":"InstanceManager","file":"illumos-utils/src/svc.rs:42"} {"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:56.244990258Z","hostname":"sock","pid":1566,"instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","component":"InstanceManager","file":"sled-agent/src/instance.rs:2077"} {"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:56.281558655Z","hostname":"sock","pid":1566,"instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","component":"InstanceManager","file":"sled-agent/src/instance.rs:2101"} {"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:56.425657394Z","hostname":"sock","pid":1566,"instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","component":"InstanceManager","file":"sled-agent/src/instance.rs:1214"} {"msg":"IGW mapping changed; external IPs refreshed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:56.426131206Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1249"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:56.426174688Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"655c9e65-3f0c-4dc5-9375-2f8d8cce2f4c","remote_addr":"[fd00:1122:3344:101::c]:41761","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":9419992,"response_code":"204"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Creating), migration_in: None, migration_out: None, time: 2025-01-15T02:37:56.427218126Z }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:56.427243967Z","hostname":"sock","pid":1566,"instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","component":"InstanceManager","file":"sled-agent/src/instance.rs:781"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:56.427260997Z","hostname":"sock","pid":1566,"instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","component":"InstanceManager","file":"sled-agent/src/instance.rs:803","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(3), time_updated: 2025-01-15T02:37:56.427218126Z }","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:56.427267127Z","hostname":"sock","pid":1566,"instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","component":"InstanceManager","file":"sled-agent/src/instance.rs:700","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Starting, gen: Generation(3), time_updated: 2025-01-15T02:37:56.427218126Z }, migration_in: None, migration_out: None }","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_in: None, migration_out: None, time: 2025-01-15T02:37:56.500370010Z }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:56.500400471Z","hostname":"sock","pid":1566,"instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","component":"InstanceManager","file":"sled-agent/src/instance.rs:781"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:56.500425881Z","hostname":"sock","pid":1566,"instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","component":"InstanceManager","file":"sled-agent/src/instance.rs:803","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(4), time_updated: 2025-01-15T02:37:56.500370010Z }","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:56.500442002Z","hostname":"sock","pid":1566,"instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","component":"InstanceManager","file":"sled-agent/src/instance.rs:700","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Running, gen: Generation(4), time_updated: 2025-01-15T02:37:56.500370010Z }, migration_in: None, migration_out: None }","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:37:56.501459569Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"aba0a651-42cc-478b-9b2d-428eee328ff7","remote_addr":"[fd00:1122:3344:101::c]:41761","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":273,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:00.754304819Z","hostname":"sock","pid":1566,"uri":"/ce66c06a-7d54-47a6-860d-59da68080fee","method":"GET","req_id":"41754de1-3c31-4b8a-ae7b-f2dd76bc0fc8","remote_addr":"[fd00:1122:3344:101::d]:56294","local_addr":"[fd00:1122:3344:101::1]:63949","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":6644,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:09.531007491Z","hostname":"sock","pid":1566,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:52014"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:09.531168316Z","hostname":"sock","pid":1566,"uri":"/baseboard","method":"GET","req_id":"fe5b2318-31b2-4593-a958-7c87bfd6c31a","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:52014","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":79,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:09.689143969Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:43446"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:09.689796046Z","hostname":"sock","pid":1566,"uri":"/vmms/04760650-1e85-40f1-9f70-557a11b03e88/state","method":"PUT","req_id":"757514aa-1129-46f5-a829-1dfc90a72fdc","remote_addr":"[fd00:1122:3344:101::c]:43446","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":565,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_in: None, migration_out: None, time: 2025-01-15T02:38:09.689911399Z }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:09.68994033Z","hostname":"sock","pid":1566,"instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","component":"InstanceManager","file":"sled-agent/src/instance.rs:781"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:09.689973601Z","hostname":"sock","pid":1566,"instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","component":"InstanceManager","file":"sled-agent/src/instance.rs:803","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(6), time_updated: 2025-01-15T02:38:09.689911399Z }","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:09.689987052Z","hostname":"sock","pid":1566,"instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","component":"InstanceManager","file":"sled-agent/src/instance.rs:700","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(6), time_updated: 2025-01-15T02:38:09.689911399Z }, migration_in: None, migration_out: None }","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_in: None, migration_out: None, time: 2025-01-15T02:38:09.709263602Z }","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:09.709279452Z","hostname":"sock","pid":1566,"instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","component":"InstanceManager","file":"sled-agent/src/instance.rs:781"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:09.709307463Z","hostname":"sock","pid":1566,"instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","component":"InstanceManager","file":"sled-agent/src/instance.rs:803","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2025-01-15T02:38:09.709263602Z }","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88"} {"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:09.709314733Z","hostname":"sock","pid":1566,"instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","component":"InstanceManager","file":"sled-agent/src/instance.rs:817","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:09.709366235Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"target removed with queued samples","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:38:09.709447657Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:602","n_samples":6,"id":"6418116445524988175"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:09.709826147Z","hostname":"sock","pid":1566,"component":"StorageManager","file":"sled-storage/src/manager.rs:607"} {"msg":"target removed with queued samples","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:38:09.70994581Z","hostname":"sock","pid":1566,"component":"kstat-sampler-worker","sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:602","n_samples":6,"id":"17935785484840642508"} {"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:09.710812383Z","hostname":"sock","pid":1566,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:339","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/04e15bc9-6d7d-41db-92b0-ff86f85636e0/debug/bundle/zone\", \"/pool/int/f20ff590-e55a-4339-aef1-379a1d2d3fc3/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/57cb94f0-4114-4bd3-970a-a0245b825f9d/crypt/debug\", \"/pool/ext/27bb8d18-eb2e-4bb2-b35c-c031637c236f/crypt/debug\", \"/pool/ext/87e49e9a-4ab3-4574-b920-463506a8978e/crypt/debug\", \"/pool/ext/b471fd39-9806-47fa-8ddb-0ca2f42aaf5d/crypt/debug\", \"/pool/ext/03ea8739-b702-4a4b-b400-0745bd6b4f96/crypt/debug\", \"/pool/ext/d8acd544-80aa-4e82-b375-eb038fe27547/crypt/debug\", \"/pool/ext/0aaa1466-164a-4540-ae41-ab17e9fc6bc8/crypt/debug\", \"/pool/ext/56c8079e-8d4e-4506-b6ff-2f0a5efab320/crypt/debug\", \"/pool/ext/276e9391-34d6-4a99-bd91-776ecada771e/crypt/debug\"] }","zone_name":"oxz_propolis-server_04760650-1e85-40f1-9f70-557a11b03e88"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:09.892310011Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:37236"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:09.92636518Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:34521"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:09.92748857Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"adc12263-8878-4a80-a0f3-b70db895c9fe","remote_addr":"[fd00:1122:3344:101::a]:34521","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":175,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:09.986235916Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {8bfad461-5ff5-4bae-8dc5-5c05bfcee1cf: {10.151.2.175: {1187a63a-5e12-4ea5-9717-553d241f7bb3}, 10.151.2.176: {1187a63a-5e12-4ea5-9717-553d241f7bb3}}} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:09.986278767Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"65a8fc40-c86a-442d-b7da-da7c9a2b8692","remote_addr":"[fd00:1122:3344:101::a]:34521","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":157,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:10.002621548Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"375823b3-f56d-4c9f-ac67-3789c1e039e6","remote_addr":"[fd00:1122:3344:101::a]:34521","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":426,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:10.187929888Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:65501"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:10.188221126Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"47fd00bd-4e53-48ad-9d69-f016ff1949e5","remote_addr":"[fd00:1122:3344:101::a]:65501","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":189,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:10.224871705Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:43118"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:10.225007558Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:10.264411212Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:10.328568563Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"86a8f26f-fc1e-449b-90bb-abb1c4bbd531","remote_addr":"[fd00:1122:3344:101::a]:43118","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":103630,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:10.328873931Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"5a9b48ef-0c8c-4c68-ab98-3a2afdad8691","remote_addr":"[fd00:1122:3344:101::a]:43118","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":107,"response_code":"200"} {"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:10.371704307Z","hostname":"sock","pid":1566,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1142","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_04760650-1e85-40f1-9f70-557a11b03e88\", bundle_id: ca731d2c-3e5a-4552-b1e0-30d599fefa32 }, time_created: 2025-01-15T02:38:09.711312797Z, version: 0, cause: TerminatedInstance }"} {"msg":"Halting and removing zone: oxz_propolis-server_04760650-1e85-40f1-9f70-557a11b03e88","v":0,"name":"SledAgent","level":40,"time":"2025-01-15T02:38:10.371774539Z","hostname":"sock","pid":1566,"instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","component":"InstanceManager","file":"sled-agent/src/instance.rs:1314"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:10.733717286Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:46952"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:10.747249631Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:34988"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:10.748139435Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"cfd6d930-284b-4403-a162-cbe5131be7be","remote_addr":"[fd00:1122:3344:101::c]:34988","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":121,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:10.78797275Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:62222"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:10.901208696Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:41020"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:10.90137882Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"aebb2083-ce03-4b69-b9a1-dc86508cc530","remote_addr":"[fd00:1122:3344:101::c]:41020","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":59,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:10.946127608Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {8bfad461-5ff5-4bae-8dc5-5c05bfcee1cf: {10.151.2.176: {1187a63a-5e12-4ea5-9717-553d241f7bb3}, 10.151.2.175: {1187a63a-5e12-4ea5-9717-553d241f7bb3}}} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:10.946165349Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"3193ed18-0c67-46fb-9cd6-f74212c1e593","remote_addr":"[fd00:1122:3344:101::c]:41020","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":105,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:10.963809855Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"ec08eb6d-ba35-474a-a46f-d5d5455c3133","remote_addr":"[fd00:1122:3344:101::c]:41020","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":211,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:11.04995515Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:57431"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:11.050022271Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:11.084953594Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:11.147624315Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"86a8573e-7f8d-4cf4-9f4d-7645c6a668f3","remote_addr":"[fd00:1122:3344:101::c]:57431","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":97623,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:11.147830251Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:63170"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:11.147880132Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:34539"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:11.147972104Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:42607"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:11.148256332Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"be4531ae-30ff-4497-8486-2930a45e08ab","remote_addr":"[fd00:1122:3344:101::c]:57431","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":279,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:11.148277883Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"6cb64395-5a7f-4e3e-9f29-387707873e39","remote_addr":"[fd00:1122:3344:101::b]:63170","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":125,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:11.148358285Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"e8d938d8-9520-4fc7-8d06-e566ea57b942","remote_addr":"[fd00:1122:3344:101::b]:42607","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":235,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:11.149946588Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4134"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:11.192008842Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {8bfad461-5ff5-4bae-8dc5-5c05bfcee1cf: {10.151.2.176: {1187a63a-5e12-4ea5-9717-553d241f7bb3}, 10.151.2.175: {1187a63a-5e12-4ea5-9717-553d241f7bb3}}} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:11.192083864Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"92b63ef1-1907-4f6f-b1a8-2634d7ffc78c","remote_addr":"[fd00:1122:3344:101::b]:63170","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":198,"response_code":"204"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:11.194657934Z","hostname":"sock","pid":1566,"component":"ServiceManager","file":"sled-agent/src/services.rs:4158"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:11.213620935Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"aba31e28-6276-4c3b-82ca-5e6538a88754","remote_addr":"[fd00:1122:3344:101::b]:63170","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":2202,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:11.261144448Z","hostname":"sock","pid":1566,"uri":"/switch-ports","method":"POST","req_id":"e9b3f284-bb50-46d6-8b86-02bcc02850a6","remote_addr":"[fd00:1122:3344:101::b]:34539","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":111279,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:11.261527898Z","hostname":"sock","pid":1566,"uri":"/network-bootstore-config","method":"GET","req_id":"bf83a2d6-9431-40e5-9593-20567d29d1f3","remote_addr":"[fd00:1122:3344:101::b]:34539","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":166,"response_code":"200"} {"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:12.163145358Z","hostname":"sock","pid":1566,"instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","component":"InstanceManager","file":"illumos-utils/src/zone.rs:279"} {"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:12.179964991Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:1000","id":"RouterId { vni: Vni(3887612), kind: System }"} {"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:12.179998482Z","hostname":"sock","pid":1566,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:1000","id":"RouterId { vni: Vni(3887612), kind: Custom(V4(Ipv4Net { addr: 172.30.0.0, width: 22 })) }"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:12.180379643Z","hostname":"sock","pid":1566,"instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","component":"InstanceManager","file":"sled-agent/src/instance.rs:700","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(8), time_updated: 2025-01-15T02:38:12.180352232Z }, migration_in: None, migration_out: None }","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:12.287208415Z","hostname":"sock","pid":1566,"instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","component":"InstanceManager","file":"sled-agent/src/instance.rs:700","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(8), time_updated: 2025-01-15T02:38:12.180352232Z }, migration_in: None, migration_out: None }","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27"} {"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:12.291969824Z","hostname":"sock","pid":1566,"instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","propolis_id":"04760650-1e85-40f1-9f70-557a11b03e88","instance_id":"62d616e1-f148-4a1c-90a4-4e962236db27","component":"InstanceManager","file":"sled-agent/src/instance.rs:1251"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:12.292169559Z","hostname":"sock","pid":1566,"uri":"/vmms/04760650-1e85-40f1-9f70-557a11b03e88/state","method":"GET","req_id":"0c0b9805-4f37-43b2-b72b-ac59724eebfc","remote_addr":"[fd00:1122:3344:101::a]:37236","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":2396717,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:12.29218717Z","hostname":"sock","pid":1566,"uri":"/vmms/04760650-1e85-40f1-9f70-557a11b03e88/state","method":"GET","req_id":"deda1b06-a019-4aa7-b730-c57111287f02","remote_addr":"[fd00:1122:3344:101::c]:46952","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":1555822,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:12.292901739Z","hostname":"sock","pid":1566,"uri":"/vmms/04760650-1e85-40f1-9f70-557a11b03e88/state","method":"GET","req_id":"edee903c-a0ae-4ee6-bb0d-659d2b72feaf","remote_addr":"[fd00:1122:3344:101::b]:62222","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":1501990,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:12.854689849Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:40761"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:12.854841383Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"534317a8-3e0a-473f-a91c-f47d725fd5e4","remote_addr":"[fd00:1122:3344:101::b]:40761","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":88,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:12.869076487Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {8bfad461-5ff5-4bae-8dc5-5c05bfcee1cf: {10.151.2.175: {1187a63a-5e12-4ea5-9717-553d241f7bb3}, 10.151.2.176: {1187a63a-5e12-4ea5-9717-553d241f7bb3}}} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:12.869097647Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"34fc51b6-cb26-497f-854a-3c8bdc4ce800","remote_addr":"[fd00:1122:3344:101::b]:40761","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":101,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:12.876501137Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"ca00909d-ea39-4320-8383-98c18284985c","remote_addr":"[fd00:1122:3344:101::b]:40761","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":167,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:12.899738554Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:64129"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:12.899915609Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"000e8c67-cf38-4a80-b8d1-48dec348b12a","remote_addr":"[fd00:1122:3344:101::b]:64129","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":106,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:13.415664286Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:54750"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:13.415869961Z","hostname":"sock","pid":1566,"uri":"/v2p","method":"GET","req_id":"94a69cf8-78cc-4375-8503-fd65e871ce06","remote_addr":"[fd00:1122:3344:101::c]:54750","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":152,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:13.4380577Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:65295"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:13.438160533Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"8e0c6b89-ea1b-4b85-ab9e-9978db00a47a","remote_addr":"[fd00:1122:3344:101::c]:65295","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":53,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:13.485525141Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"IGW mapping changed; external IPs refreshed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:13.485584273Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1249"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:13.485597053Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"2482e59f-bb6e-495c-8222-cde08819525a","remote_addr":"[fd00:1122:3344:101::c]:65295","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":132,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:13.493974999Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"79237784-a4e4-411f-b9e2-52e19162dd89","remote_addr":"[fd00:1122:3344:101::c]:65295","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":296,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:14.412748241Z","hostname":"sock","pid":1566,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:58486"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:14.412922506Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"GET","req_id":"90877999-44fd-4baf-90d1-6b2fb8e38fbc","remote_addr":"[fd00:1122:3344:101::c]:58486","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":94,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:14.454622561Z","hostname":"sock","pid":1566,"sled_id":"ce66c06a-7d54-47a6-860d-59da68080fee","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1223","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:14.454650802Z","hostname":"sock","pid":1566,"uri":"/eip-gateways","method":"PUT","req_id":"97b0c7af-7dbb-46f2-afc5-254172816447","remote_addr":"[fd00:1122:3344:101::c]:58486","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":84,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2025-01-15T02:38:14.462548485Z","hostname":"sock","pid":1566,"uri":"/vpc-routes","method":"PUT","req_id":"56376d3f-bb5a-4084-8023-4efd38b4f671","remote_addr":"[fd00:1122:3344:101::c]:58486","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":201,"response_code":"204"}