[ Nov 18 20:58:01 Enabled. ] [ Nov 18 20:58:01 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ Nov 18 20:58:01 Method "start" exited with status 0. ] [ Nov 18 20:58:01 Rereading configuration. ] [ Nov 18 20:58:01 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:01.461647516Z","hostname":"sock","pid":1574,"file":"illumos-utils/src/running_zone.rs:113"} {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:01.461821717Z","hostname":"sock","pid":1574,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:01.513769514Z","hostname":"sock","pid":1574,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:01.513806885Z","hostname":"sock","pid":1574,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:01.627846775Z","hostname":"sock","pid":1574,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:01.636715582Z","hostname":"sock","pid":1574,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:01.644422439Z","hostname":"sock","pid":1574,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:02.246306717Z","hostname":"sock","pid":1574,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fdb0:18c0:4d0d:a02a::, width: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:02.38334415Z","hostname":"sock","pid":1574,"file":"sled-agent/src/long_running_tasks.rs:126"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:02.38337741Z","hostname":"sock","pid":1574,"file":"sled-agent/src/long_running_tasks.rs:138"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:02.38342575Z","hostname":"sock","pid":1574,"file":"sled-agent/src/long_running_tasks.rs:151"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:02.383492771Z","hostname":"sock","pid":1574,"file":"sled-agent/src/long_running_tasks.rs:173","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:02.383520251Z","hostname":"sock","pid":1574,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:717"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:02.383538191Z","hostname":"sock","pid":1574,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:520"} {"msg":"Generated i86pc baseboard Pc { identifier: \"sock\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:02.386898606Z","hostname":"sock","pid":1574,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:624"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:02.386905026Z","hostname":"sock","pid":1574,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:766"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:02.386919436Z","hostname":"sock","pid":1574,"file":"sled-agent/src/long_running_tasks.rs:187"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:02.386929176Z","hostname":"sock","pid":1574,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:825"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:02.386956097Z","hostname":"sock","pid":1574,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_0.vdev"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:02.386980057Z","hostname":"sock","pid":1574,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"None"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:02.387087578Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:02.387116298Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:02.387141108Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:02.39815438Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:02.39818145Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-hardware/src/disk.rs:378","id":"b7347e5b-0882-41c4-bf1e-7a5fbf6725b3 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:02.497561582Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: b7347e5b-0882-41c4-bf1e-7a5fbf6725b3 (zpool), kind: Internal }"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:02.969842044Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:02.969891374Z","hostname":"sock","pid":1574,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/m2_1.vdev"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:02.969897694Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:02.969913054Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:02.969925104Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:02.969936964Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:02.969942164Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:02.978623559Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:02.978646379Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-hardware/src/disk.rs:378","id":"32b20b70-d4a3-45a0-a7b8-ec67799d407c (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.071324541Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 32b20b70-d4a3-45a0-a7b8-ec67799d407c (zpool), kind: Internal }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.547788054Z","hostname":"sock","pid":1574,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_0.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.547832155Z","hostname":"sock","pid":1574,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: b7347e5b-0882-41c4-bf1e-7a5fbf6725b3 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 32b20b70-d4a3-45a0-a7b8-ec67799d407c (zpool), kind: Internal } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.547871635Z","hostname":"sock","pid":1574,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.547878315Z","hostname":"sock","pid":1574,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.547887695Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.547893815Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.547905785Z","hostname":"sock","pid":1574,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_1.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.547917525Z","hostname":"sock","pid":1574,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: b7347e5b-0882-41c4-bf1e-7a5fbf6725b3 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 32b20b70-d4a3-45a0-a7b8-ec67799d407c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.547932285Z","hostname":"sock","pid":1574,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.547936785Z","hostname":"sock","pid":1574,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.547941785Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.547950475Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.547955905Z","hostname":"sock","pid":1574,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_2.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.547967585Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.547976305Z","hostname":"sock","pid":1574,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: b7347e5b-0882-41c4-bf1e-7a5fbf6725b3 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 32b20b70-d4a3-45a0-a7b8-ec67799d407c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.547997196Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548004836Z","hostname":"sock","pid":1574,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548012036Z","hostname":"sock","pid":1574,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548016986Z","hostname":"sock","pid":1574,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_3.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548022636Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548029736Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548035326Z","hostname":"sock","pid":1574,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: b7347e5b-0882-41c4-bf1e-7a5fbf6725b3 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 32b20b70-d4a3-45a0-a7b8-ec67799d407c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548058806Z","hostname":"sock","pid":1574,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_4.vdev"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548063636Z","hostname":"sock","pid":1574,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548070906Z","hostname":"sock","pid":1574,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548076646Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548081886Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548088526Z","hostname":"sock","pid":1574,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: b7347e5b-0882-41c4-bf1e-7a5fbf6725b3 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 32b20b70-d4a3-45a0-a7b8-ec67799d407c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548112077Z","hostname":"sock","pid":1574,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548117277Z","hostname":"sock","pid":1574,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548123207Z","hostname":"sock","pid":1574,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_5.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548127807Z","hostname":"sock","pid":1574,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: b7347e5b-0882-41c4-bf1e-7a5fbf6725b3 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 32b20b70-d4a3-45a0-a7b8-ec67799d407c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548160527Z","hostname":"sock","pid":1574,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548167677Z","hostname":"sock","pid":1574,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548173127Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548182107Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548187597Z","hostname":"sock","pid":1574,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_6.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548193527Z","hostname":"sock","pid":1574,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: b7347e5b-0882-41c4-bf1e-7a5fbf6725b3 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 32b20b70-d4a3-45a0-a7b8-ec67799d407c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548223527Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548228397Z","hostname":"sock","pid":1574,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548235918Z","hostname":"sock","pid":1574,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548240498Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548245767Z","hostname":"sock","pid":1574,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_7.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548253087Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548258388Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548263658Z","hostname":"sock","pid":1574,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: b7347e5b-0882-41c4-bf1e-7a5fbf6725b3 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 32b20b70-d4a3-45a0-a7b8-ec67799d407c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548313848Z","hostname":"sock","pid":1574,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548321988Z","hostname":"sock","pid":1574,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548326778Z","hostname":"sock","pid":1574,"file":"sled-agent/src/long_running_tasks.rs:243","vdev":"/scratch/u2_8.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548332648Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548337508Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:457","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548343038Z","hostname":"sock","pid":1574,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: b7347e5b-0882-41c4-bf1e-7a5fbf6725b3 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 32b20b70-d4a3-45a0-a7b8-ec67799d407c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548371488Z","hostname":"sock","pid":1574,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548377188Z","hostname":"sock","pid":1574,"file":"sled-agent/src/long_running_tasks.rs:97"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548385168Z","hostname":"sock","pid":1574,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548397449Z","hostname":"sock","pid":1574,"file":"sled-agent/src/long_running_tasks.rs:99"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548402609Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548407719Z","hostname":"sock","pid":1574,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: b7347e5b-0882-41c4-bf1e-7a5fbf6725b3 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 32b20b70-d4a3-45a0-a7b8-ec67799d407c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548437329Z","hostname":"sock","pid":1574,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548442959Z","hostname":"sock","pid":1574,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548447899Z","hostname":"sock","pid":1574,"file":"sled-agent/src/long_running_tasks.rs:211"} {"msg":"No ledger in /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548452929Z","hostname":"sock","pid":1574,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:189"} {"msg":"No ledger in /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548459759Z","hostname":"sock","pid":1574,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:189"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548464999Z","hostname":"sock","pid":1574,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548478049Z","hostname":"sock","pid":1574,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:189"} {"msg":"No ledger in /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548483859Z","hostname":"sock","pid":1574,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:189"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548489539Z","hostname":"sock","pid":1574,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548494859Z","hostname":"sock","pid":1574,"file":"sled-agent/src/long_running_tasks.rs:216"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.548499369Z","hostname":"sock","pid":1574,"file":"sled-agent/src/long_running_tasks.rs:231"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.559549592Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:743"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.559584882Z","hostname":"sock","pid":1574,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:186","time_to_next_cleanup":"599.99994206s","next_cleanup":"Instant { tv_sec: 1392, tv_nsec: 613436126 }"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.559627483Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4000"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.559640492Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"No ledger in /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.559695203Z","hostname":"sock","pid":1574,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:189"} {"msg":"No ledger in /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.562606725Z","hostname":"sock","pid":1574,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:189"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.562966177Z","hostname":"sock","pid":1574,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.564922392Z","hostname":"sock","pid":1574,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:51","local_addr":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.2.164,\n last: 10.151.2.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.2.164,\n 10.151.2.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.2.174,\n infra_ip_last: 10.151.2.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.2.100,\n vlan_id: None,\n rib_priority: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.2.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: Some(\n None,\n ),\n bgp_peers: [],\n autoneg: false,\n lldp: None,\n tx_eq: None,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n}","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.565261685Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1200"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.565276905Z","hostname":"sock","pid":1574,"component":"DnsResolver","component":"RSS","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.565324905Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"No ledger in /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.565384395Z","hostname":"sock","pid":1574,"component":"RSS","file":"common/src/ledger.rs:189"} {"msg":"No ledger in /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.565404946Z","hostname":"sock","pid":1574,"component":"RSS","file":"common/src/ledger.rs:189"} {"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.565412706Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1263"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.565433966Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"No ledger in /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.565449586Z","hostname":"sock","pid":1574,"component":"RSS","file":"common/src/ledger.rs:189"} {"msg":"No ledger in /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.565460786Z","hostname":"sock","pid":1574,"component":"RSS","file":"common/src/ledger.rs:189"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.565471126Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1305"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.565476286Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:156"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:a02a::1","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.565481256Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:129"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.565493666Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Writing ledger to /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.565522096Z","hostname":"sock","pid":1574,"component":"RSS","file":"common/src/ledger.rs:200"} {"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.565837339Z","hostname":"sock","pid":1574,"component":"RSS","file":"common/src/ledger.rs:133","path":"\"/pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/rss-sled-plan.json\""} {"msg":"Writing ledger to /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.565845599Z","hostname":"sock","pid":1574,"component":"RSS","file":"common/src/ledger.rs:200"} {"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.56598552Z","hostname":"sock","pid":1574,"component":"RSS","file":"common/src/ledger.rs:133","path":"\"/pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/rss-sled-plan.json\""} {"msg":"Sled plan written to storage: Plan {\n rack_id: f35dae23-a43a-4d77-a204-fab3c4bdfff3,\n sleds: {\n [fdb0:18c0:4d0d:a02a::1]:12346: StartSledAgentRequest {\n generation: 0,\n schema_version: 1,\n body: StartSledAgentRequestBody {\n id: 79cbda5a-2e27-489e-945d-860d86dec819 (sled),\n rack_id: f35dae23-a43a-4d77-a204-fab3c4bdfff3,\n use_trust_quorum: false,\n is_lrtq_learner: false,\n subnet: Ipv6Subnet {\n net: Ipv6Net {\n addr: fd00:1122:3344:101::,\n width: 64,\n },\n },\n },\n },\n },\n config: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.2.164,\n last: 10.151.2.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.2.164,\n 10.151.2.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.2.174,\n infra_ip_last: 10.151.2.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.2.100,\n vlan_id: None,\n rib_priority: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.2.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: Some(\n None,\n ),\n bgp_peers: [],\n autoneg: false,\n lldp: None,\n tx_eq: None,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.56608106Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:176"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.566088531Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1339"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.566116811Z","hostname":"sock","pid":1574,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.566129631Z","hostname":"sock","pid":1574,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:200"} {"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.566290872Z","hostname":"sock","pid":1574,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:133","path":"\"/pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/cluster/bootstore-network-config.json\""} {"msg":"Writing ledger to /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.566298392Z","hostname":"sock","pid":1574,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:200"} {"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.566445743Z","hostname":"sock","pid":1574,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:133","path":"\"/pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/cluster/bootstore-network-config.json\""} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.566452813Z","hostname":"sock","pid":1574,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.566489724Z","hostname":"sock","pid":1574,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 79cbda5a-2e27-489e-945d-860d86dec819 (sled), rack_id: f35dae23-a43a-4d77-a204-fab3c4bdfff3, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.566811026Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4297"} {"msg":"Starting switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.566873466Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4697"} {"msg":"Loaded keys and certs","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.571916574Z","hostname":"sock","pid":1574,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:18c0:4d0d:a02a::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/client.rs:50"} {"msg":"Certificate chain verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.578356922Z","hostname":"sock","pid":1574,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:18c0:4d0d:a02a::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:307"} {"msg":"Signature verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.578508863Z","hostname":"sock","pid":1574,"component":"SledAgentSprocketsClient","BootstrapAgentClient":"[fdb0:18c0:4d0d:a02a::1]:12346","component":"BootstrapAgentRssHandler","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:376"} {"msg":"Certificate chain verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.582780185Z","hostname":"sock","pid":1574,"component":"SledAgentSprocketsServer","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:307"} {"msg":"Signature verified successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.582840325Z","hostname":"sock","pid":1574,"component":"SledAgentSprocketsServer","file":"/home/build/.cargo/git/checkouts/sprockets-82488b5b61bf00db/cc13773/tls/src/keys.rs:376"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.582945356Z","hostname":"sock","pid":1574,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:57907","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:74"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 79cbda5a-2e27-489e-945d-860d86dec819 (sled), rack_id: f35dae23-a43a-4d77-a204-fab3c4bdfff3, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.583077637Z","hostname":"sock","pid":1574,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:362"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.583089707Z","hostname":"sock","pid":1574,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:377"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.583122428Z","hostname":"sock","pid":1574,"file":"sled-agent/src/server.rs:44"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.583127588Z","hostname":"sock","pid":1574,"component":"DnsResolver","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.583153898Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.583158968Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:809"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.583184448Z","hostname":"sock","pid":1574,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:101::, width: 64 }"} {"msg":"No ledger in /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.583199958Z","hostname":"sock","pid":1574,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:189"} {"msg":"No ledger in /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.583213418Z","hostname":"sock","pid":1574,"request":"load_disks_ledger","component":"StorageManager","file":"common/src/ledger.rs:189"} {"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.583218478Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:798"} {"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.583230019Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:826"} {"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.583235879Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:592"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.583266639Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"} {"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.583274119Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:697","result":"DisksManagementResult { status: [] }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.600533468Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.621102721Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.63839639Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.6599383Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:407"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.660021981Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.660029421Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:419"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.660038371Z","hostname":"sock","pid":1574,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 91.921419ms)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.663088414Z","hostname":"sock","pid":1574,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c3f70a77-a2f6-4016-a447-665592c1143e\", \"content-length\": \"124\", \"date\": \"Mon, 18 Nov 2024 20:58:03 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c3f70a77-a2f6-4016-a447-665592c1143e\" }"} {"msg":"attempting to create encrypted zvol: name=\"oxi_b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.674022436Z","hostname":"sock","pid":1574,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"successfully created encrypted zvol: name=\"oxi_b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.697474871Z","hostname":"sock","pid":1574,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.697520601Z","hostname":"sock","pid":1574,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.70279334Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:434"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.70280384Z","hostname":"sock","pid":1574,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:128"} {"msg":"Ensuring dataset oxi_b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.70280925Z","hostname":"sock","pid":1574,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:138"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.755783495Z","hostname":"sock","pid":1574,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 127.59184ms)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.75635895Z","hostname":"sock","pid":1574,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0c9717f9-7fbd-4849-8912-ec9cf434947b\", \"content-length\": \"124\", \"date\": \"Mon, 18 Nov 2024 20:58:03 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0c9717f9-7fbd-4849-8912-ec9cf434947b\" }"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.785452797Z","hostname":"sock","pid":1574,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:173"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.884601556Z","hostname":"sock","pid":1574,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 125.988117ms)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:03.88519026Z","hostname":"sock","pid":1574,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fab9bac3-6533-4d1c-90c9-50274c02f0af\", \"content-length\": \"124\", \"date\": \"Mon, 18 Nov 2024 20:58:03 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"fab9bac3-6533-4d1c-90c9-50274c02f0af\" }"} {"msg":"Mounting oxi_b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:04.003647224Z","hostname":"sock","pid":1574,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:181"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:04.012739782Z","hostname":"sock","pid":1574,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 152.878173ms)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:04.013266056Z","hostname":"sock","pid":1574,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7d8b6cc3-bca7-4ca8-a9b7-07de6b38a5e2\", \"content-length\": \"124\", \"date\": \"Mon, 18 Nov 2024 20:58:03 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7d8b6cc3-bca7-4ca8-a9b7-07de6b38a5e2\" }"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:04.01792563Z","hostname":"sock","pid":1574,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:189"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:04.018030211Z","hostname":"sock","pid":1574,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:189"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:04.018052971Z","hostname":"sock","pid":1574,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:189"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:04.018094422Z","hostname":"sock","pid":1574,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:195"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:04.167279784Z","hostname":"sock","pid":1574,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 216.835263ms)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:04.16803399Z","hostname":"sock","pid":1574,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"afe5f5aa-adcf-4485-bbb3-b6a1f685fed2\", \"content-length\": \"124\", \"date\": \"Mon, 18 Nov 2024 20:58:03 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"afe5f5aa-adcf-4485-bbb3-b6a1f685fed2\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:04.385392581Z","hostname":"sock","pid":1574,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 186.617916ms)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:04.386198367Z","hostname":"sock","pid":1574,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4ff4bc8a-61c7-41bb-84e7-84fe2abb0629\", \"content-length\": \"124\", \"date\": \"Mon, 18 Nov 2024 20:58:03 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4ff4bc8a-61c7-41bb-84e7-84fe2abb0629\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:04.573255492Z","hostname":"sock","pid":1574,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 399.447079ms)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:04.573688246Z","hostname":"sock","pid":1574,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7a0879fd-0bc1-4f78-9aab-f17793e68e29\", \"content-length\": \"124\", \"date\": \"Mon, 18 Nov 2024 20:58:03 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7a0879fd-0bc1-4f78-9aab-f17793e68e29\" }"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:04.750083141Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:93"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:04.755508382Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:58808","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327","component":"producer-server","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:04.755543612Z","hostname":"sock","pid":1574,"component":"internal-dns-resolver","file":"internal-dns/resolver/src/resolver.rs:111","component":"producer-server","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:04.755590402Z","hostname":"sock","pid":1574,"file":"oximeter/producer/src/lib.rs:279","component":"producer-server","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/producer/src/lib.rs:279","interval":"30s","address":"[fd00:1122:3344:101::1]:58808","producer_id":"79cbda5a-2e27-489e-945d-860d86dec819"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:04.756772381Z","hostname":"sock","pid":1574,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"355.115963ms"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:04.757646647Z","hostname":"sock","pid":1574,"component":"VmmReservoirManager","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:04.757661768Z","hostname":"sock","pid":1574,"component":"VmmReservoirManager","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:04.757714798Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:927","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:04.757720288Z","hostname":"sock","pid":1574,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/resolver/src/resolver.rs:111","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:04.973766999Z","hostname":"sock","pid":1574,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:05.113129189Z","hostname":"sock","pid":1574,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"274.491272ms"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:05.389727452Z","hostname":"sock","pid":1574,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"1.317183016s"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:06.7089017Z","hostname":"sock","pid":1574,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"2.450007612s"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:07.33246058Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:2675"} {"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:07.33249306Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:2920"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:07.332498261Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:2545"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:07.33250249Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:2592"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:07.33251207Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:07.332516441Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3046"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:07.334474355Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:2845"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:07.335794735Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:2586"} {"msg":"Profile for oxz_switch:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:07.335895356Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:07.336049077Z","hostname":"sock","pid":1574,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 27.200137ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:07.586455035Z","hostname":"sock","pid":1574,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 84.853021ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:07.777718131Z","hostname":"sock","pid":1574,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 128.290497ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:07.974235506Z","hostname":"sock","pid":1574,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 433.763965ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:08.169661314Z","hostname":"sock","pid":1574,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 688.663631ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:08.785348476Z","hostname":"sock","pid":1574,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:09.160735725Z","hostname":"sock","pid":1574,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"5.357355543s"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 659.467621ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:09.597443133Z","hostname":"sock","pid":1574,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 579.13497ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:10.427426392Z","hostname":"sock","pid":1574,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.22000401s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:11.049016078Z","hostname":"sock","pid":1574,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.495285807s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:12.332629352Z","hostname":"sock","pid":1574,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/79cbda5a-2e27-489e-945d-860d86dec819)","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:14.241457437Z","hostname":"sock","pid":1574,"component":"NexusNotifierTask","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/nexus.rs:527"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.27123845Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.271574092Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12348","component":"dropshot (Repo Depot)","component":"ArtifactStore","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.272449629Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:327"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.272472319Z","hostname":"sock","pid":1574,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:432"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.272480479Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Writing ledger to /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.272493749Z","hostname":"sock","pid":1574,"component":"SledAgentMain","file":"common/src/ledger.rs:200"} {"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.2726512Z","hostname":"sock","pid":1574,"component":"SledAgentMain","file":"common/src/ledger.rs:133","path":"\"/pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/sled-agent-request.json\""} {"msg":"Writing ledger to /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.27265649Z","hostname":"sock","pid":1574,"component":"SledAgentMain","file":"common/src/ledger.rs:200"} {"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.272773191Z","hostname":"sock","pid":1574,"component":"SledAgentMain","file":"common/src/ledger.rs:133","path":"\"/pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/sled-agent-request.json\""} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.272811111Z","hostname":"sock","pid":1574,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.272853882Z","hostname":"sock","pid":1574,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV2 { rack_subnet: Ipv6Net { addr: fd00:1122:3344:100::, width: 56 }, infra_ip_first: 10.151.2.174, infra_ip_last: 10.151.2.174, ports: [PortConfigV2 { routes: [RouteConfig { destination: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), nexthop: 10.151.2.100, vlan_id: None, rib_priority: None }], addresses: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: Some(None), bgp_peers: [], autoneg: false, lldp: None, tx_eq: None }], bgp: [], bfd: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.272863992Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4000"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.272903392Z","hostname":"sock","pid":1574,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:57907","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:81"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.456538762Z","hostname":"sock","pid":1574,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:117","peer":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.456635613Z","hostname":"sock","pid":1574,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:252","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.456642922Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"No ledger in /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/rss-service-plan-v5.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.456697543Z","hostname":"sock","pid":1574,"component":"RSS","file":"common/src/ledger.rs:189"} {"msg":"No ledger in /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/rss-service-plan-v5.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.456704143Z","hostname":"sock","pid":1574,"component":"RSS","file":"common/src/ledger.rs:189"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.456734443Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.456758303Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.456790184Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.456801794Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.460857294Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4321","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.460899174Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4337"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.460908354Z","hostname":"sock","pid":1574,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:534"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:14.671944458Z","hostname":"sock","pid":1574,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"5.653398574s"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.699912547Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:46436"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.700155169Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.700163279Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"No ledger in /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.700183859Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:189"} {"msg":"No ledger in /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.700192609Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:189"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.70028305Z","hostname":"sock","pid":1574,"uri":"/inventory","method":"GET","req_id":"94eaa590-cda1-4615-88ea-7358eeae8d48","remote_addr":"[fd00:1122:3344:101::1]:46436","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":234,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.731904706Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:48950"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.732074837Z","hostname":"sock","pid":1574,"uri":"/sled-role","method":"GET","req_id":"5c8774ee-f96a-44f9-8b67-1a2d11610498","remote_addr":"[fd00:1122:3344:101::1]:48950","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":50,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.733659739Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Writing ledger to /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/.rss-service-plan-v5.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.733710729Z","hostname":"sock","pid":1574,"component":"RSS","file":"common/src/ledger.rs:200"} {"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.734100342Z","hostname":"sock","pid":1574,"component":"RSS","file":"common/src/ledger.rs:133","path":"\"/pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/rss-service-plan-v5.json\""} {"msg":"Writing ledger to /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/.rss-service-plan-v5.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.734105282Z","hostname":"sock","pid":1574,"component":"RSS","file":"common/src/ledger.rs:200"} {"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.734467065Z","hostname":"sock","pid":1574,"component":"RSS","file":"common/src/ledger.rs:133","path":"\"/pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/rss-service-plan-v5.json\""} {"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.734471965Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:980"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-18T20:58:14.760754361Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12123728031068596224"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-18T20:58:14.760845941Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"11959324033292328996"} {"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: f1262df0-ec0d-496c-9e6a-e0c18d121951, pool_id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: b65f0e6e-33c6-4d78-9169-a2fc4a525ee4, pool_id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: 16c7fbdc-a171-4925-90bc-828974eb505a, pool_id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 92537496-357c-4896-9c56-3c2e4dec320e, pool_id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: fccd86d3-1527-4d30-8d7d-92b6be90b8cb, pool_id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: f8b30b1c-b0ac-46bd-9d88-c790a12f5a02, pool_id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: 7912d971-2ec3-4443-9bcf-b0fc92399d67, pool_id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 5a2821c3-8662-450f-a7e4-5d683ece42c6, pool_id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: 56c3f052-35bb-450f-875e-b9f9c629209c, pool_id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool) }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.763672163Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:382"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.763768903Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:38753"} {"msg":"physical disks ensure","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.763940965Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:883"} {"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: f1262df0-ec0d-496c-9e6a-e0c18d121951, pool_id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: b65f0e6e-33c6-4d78-9169-a2fc4a525ee4, pool_id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: 16c7fbdc-a171-4925-90bc-828974eb505a, pool_id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 92537496-357c-4896-9c56-3c2e4dec320e, pool_id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: fccd86d3-1527-4d30-8d7d-92b6be90b8cb, pool_id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: f8b30b1c-b0ac-46bd-9d88-c790a12f5a02, pool_id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: 7912d971-2ec3-4443-9bcf-b0fc92399d67, pool_id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 5a2821c3-8662-450f-a7e4-5d683ece42c6, pool_id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: 56c3f052-35bb-450f-875e-b9f9c629209c, pool_id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.763960245Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"No ledger in /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.763973495Z","hostname":"sock","pid":1574,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:189"} {"msg":"No ledger in /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.763979015Z","hostname":"sock","pid":1574,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:189"} {"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.763983815Z","hostname":"sock","pid":1574,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:1175"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.763988655Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:302"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.763993275Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.763998695Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.764003085Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.887052153Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:14.887083603Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"473f0b4e-a714-496d-8d5d-665782986893 (zpool)"} {"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:15.097522142Z","hostname":"sock","pid":1574,"component":"VmmReservoirManager","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:15.19248821Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:15.20044586Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:240","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:15.20050812Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:242","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:15.20052287Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:246","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:15.20054633Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:248","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:15.200568501Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:15.200720172Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:15.200753342Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:260"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:15.236073915Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_473f0b4e-a714-496d-8d5d-665782986893/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:15.243824083Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:303"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/79cbda5a-2e27-489e-945d-860d86dec819)","v":0,"name":"SledAgent","level":50,"time":"2024-11-18T20:58:15.274005208Z","hostname":"sock","pid":1574,"component":"ProbeManager","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Looking for unencrypted datasets in oxp_473f0b4e-a714-496d-8d5d-665782986893","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:15.413894532Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:389"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:15.421585589Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:15.421623889Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:15.421638199Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:15.42166613Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:15.42167558Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:15.430839038Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:15.430872108Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:15.520218195Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:15.527066686Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:240","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:15.527104516Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:242","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:15.527114046Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:246","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:15.527178297Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:248","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:15.527193257Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:15.527287527Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:15.527295138Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:260"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:15.558104197Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:15.565074269Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:303"} {"msg":"Looking for unencrypted datasets in oxp_3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:15.739888863Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:389"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:15.747919613Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:15.747957103Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:15.747962303Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:15.747970543Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:15.747976833Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:15.75821782Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:15.7582543Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:15.853636872Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","zpool":"ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:15.861361939Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:240","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:15.861395019Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:242","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:15.861402829Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:246","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:15.861421339Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:248","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:15.861430959Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:15.86153806Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:15.861545Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:260"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:15.89634896Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:15.904291769Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:303"} {"msg":"Looking for unencrypted datasets in oxp_bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.082514848Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:389"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.090082255Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.090131675Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.090139795Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.090147885Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.090154925Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.099588035Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.099616566Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.191992575Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","zpool":"ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.200253226Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:240","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.200292217Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:242","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.200299537Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:246","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.200306397Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:248","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.200317177Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.200453198Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.200469388Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:260"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.234730163Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_2536a198-fc1d-4175-aab5-a62f1506f3e4/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.243302047Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:303"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/79cbda5a-2e27-489e-945d-860d86dec819)","v":0,"name":"SledAgent","level":50,"time":"2024-11-18T20:58:16.275532438Z","hostname":"sock","pid":1574,"component":"ProbeManager","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Looking for unencrypted datasets in oxp_2536a198-fc1d-4175-aab5-a62f1506f3e4","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.412427809Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:389"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.420220327Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.420245347Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.420251517Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.420257027Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.420261807Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.429880039Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.429899599Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.525993526Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","zpool":"ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.534067726Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:240","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.534125696Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:242","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.534136656Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:246","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.534169317Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:248","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.534185107Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.534365638Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.534372988Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:260"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.570601488Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_dd78d2d6-0d80-45fe-9f73-e75ef7da86ee/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.578348006Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:303"} {"msg":"Looking for unencrypted datasets in oxp_dd78d2d6-0d80-45fe-9f73-e75ef7da86ee","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.753301011Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:389"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.760928628Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.760965248Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.760980878Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.760986779Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.760991568Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.771081734Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.771102114Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.861892161Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","zpool":"ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.868723872Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:240","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.868752442Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:242","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.868784263Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:246","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.868952104Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:248","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.868962334Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.869079705Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.869093895Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:260"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.905227824Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_28b058a3-a4b5-473e-8580-f5c606b6d30b/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:16.913068673Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:303"} {"msg":"Looking for unencrypted datasets in oxp_28b058a3-a4b5-473e-8580-f5c606b6d30b","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.093343717Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:389"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.101395387Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.101439167Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.101445157Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.101449937Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.101464127Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.111397812Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.111424232Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.208745178Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","zpool":"ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.216780867Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:240","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.216840328Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:242","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.216850058Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:246","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.216869848Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:248","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.216880498Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.217030439Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.217041249Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:260"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.253063968Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_80f7cc50-8d94-4fb0-b296-446e3acf3366/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.260893987Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:303"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/79cbda5a-2e27-489e-945d-860d86dec819)","v":0,"name":"SledAgent","level":50,"time":"2024-11-18T20:58:17.278534988Z","hostname":"sock","pid":1574,"component":"ProbeManager","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Looking for unencrypted datasets in oxp_80f7cc50-8d94-4fb0-b296-446e3acf3366","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.442946074Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:389"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.450950524Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.450990804Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.451000714Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.451008385Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.451019465Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.460692237Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.460723307Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.557438298Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","zpool":"ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.565295557Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:240","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.565325167Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:242","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.565344597Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:246","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.565393908Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:248","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.565406778Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.565546109Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.565552489Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:260"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.603815864Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_feac0a3f-e4dc-4a16-9eeb-839f74ad9665/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.611814434Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:303"} {"msg":"Looking for unencrypted datasets in oxp_feac0a3f-e4dc-4a16-9eeb-839f74ad9665","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.786624208Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:389"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.794591807Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.794626377Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.794664608Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:342","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.794670968Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:419"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.794677628Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.804213959Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.804253289Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-hardware/src/disk.rs:373","id":"f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.90225211Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:196","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","zpool":"ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.910193809Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:240","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.910229599Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:242","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.910235759Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:246","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.91025364Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:248","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.91026213Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.910385161Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.910392081Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:260"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.946106077Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_f334d74e-134c-4206-a5ba-9a6ac6ea7954/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:17.953831795Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:303"} {"msg":"Looking for unencrypted datasets in oxp_f334d74e-134c-4206-a5ba-9a6ac6ea7954","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:18.125704206Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/dataset.rs:389"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:18.133454384Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:440","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:18.133500014Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:357","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Writing ledger to /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:18.133578465Z","hostname":"sock","pid":1574,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:200"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:18.133716386Z","hostname":"sock","pid":1574,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: b7347e5b-0882-41c4-bf1e-7a5fbf6725b3 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 32b20b70-d4a3-45a0-a7b8-ec67799d407c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, number_of_slots: 1, slot_firmware_versions: [Some(\"SYNTH1\")] } }, zpool_name: ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:18.133774626Z","hostname":"sock","pid":1574,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:18.133788466Z","hostname":"sock","pid":1574,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:18.133865197Z","hostname":"sock","pid":1574,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:133","path":"\"/pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-physical-disks.json\""} {"msg":"Writing ledger to /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:18.133872437Z","hostname":"sock","pid":1574,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:200"} {"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:18.134043238Z","hostname":"sock","pid":1574,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:133","path":"\"/pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-physical-disks.json\""} {"msg":"physical disks ensure: Updated storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:18.134086928Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:887"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:18.134094619Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"physical disks ensure: Propagating new generation of disks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:18.134103799Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:913","generation":"Generation(1)"} {"msg":"physical disks ensure: Updated storage monitor","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:18.134109279Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:918"} {"msg":"physical disks ensure: Updated zone bundler","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:18.134113309Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:923"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:18.134140349Z","hostname":"sock","pid":1574,"component":"ProbeManager","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:122","gen":"Generation(1)"} {"msg":"physical disks ensure: Updated probes","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:18.134148099Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:929"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:18.134179119Z","hostname":"sock","pid":1574,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:799","gen":"Generation(1)"} {"msg":"physical disks ensure: Updated instances","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:18.134192009Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:934"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:18.13426942Z","hostname":"sock","pid":1574,"uri":"/omicron-physical-disks","method":"PUT","req_id":"68edf24d-59d2-4bed-8f17-5f8b7320c3c5","remote_addr":"[fd00:1122:3344:101::1]:38753","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":3370449,"response_code":"200"} {"msg":"attempting to set up sled's datasets: DatasetsConfig { generation: Generation(1), datasets: {03f95c58-edde-4ef2-b4e3-cc6750a67b6a (dataset): DatasetConfig { id: 03f95c58-edde-4ef2-b4e3-cc6750a67b6a (dataset), name: DatasetName { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 08d01d66-7e43-4073-b6d7-ea6917fea0a8 (dataset): DatasetConfig { id: 08d01d66-7e43-4073-b6d7-ea6917fea0a8 (dataset), name: DatasetName { pool_name: ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 098371c0-6e51-482e-a7d7-0162e882f407 (dataset): DatasetConfig { id: 098371c0-6e51-482e-a7d7-0162e882f407 (dataset), name: DatasetName { pool_name: ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 0cd5c4d9-d23c-49bd-8eac-8c52fafbc7ef (dataset): DatasetConfig { id: 0cd5c4d9-d23c-49bd-8eac-8c52fafbc7ef (dataset), name: DatasetName { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 0d738e08-d7ba-4c02-b6d7-1720dbda5ab8 (dataset): DatasetConfig { id: 0d738e08-d7ba-4c02-b6d7-1720dbda5ab8 (dataset), name: DatasetName { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_62e83095-0557-4ac0-8b12-7429e79e4bb6\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 14483828-0a7a-41c4-a00f-141ccc8c88b5 (dataset): DatasetConfig { id: 14483828-0a7a-41c4-a00f-141ccc8c88b5 (dataset), name: DatasetName { pool_name: ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }, kind: TransientZone { name: \"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 15039f4a-dcb6-44bc-a0b5-f12bb083a809 (dataset): DatasetConfig { id: 15039f4a-dcb6-44bc-a0b5-f12bb083a809 (dataset), name: DatasetName { pool_name: ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 15c7ae7f-8366-4e2d-873c-4cc3ee02a5fe (dataset): DatasetConfig { id: 15c7ae7f-8366-4e2d-873c-4cc3ee02a5fe (dataset), name: DatasetName { pool_name: ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 1ae1a305-21b6-43f9-a045-d321285817f7 (dataset): DatasetConfig { id: 1ae1a305-21b6-43f9-a045-d321285817f7 (dataset), name: DatasetName { pool_name: ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 2039e497-4e63-4c59-8e48-d06006699ec6 (dataset): DatasetConfig { id: 2039e497-4e63-4c59-8e48-d06006699ec6 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 207aaac7-e112-4299-b3eb-05f2e29a65e6 (dataset): DatasetConfig { id: 207aaac7-e112-4299-b3eb-05f2e29a65e6 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_4554ce4c-90f4-4fd2-93bd-086596238122\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 20c72dfc-63f3-46d8-bbbd-4f5a7772c9fc (dataset): DatasetConfig { id: 20c72dfc-63f3-46d8-bbbd-4f5a7772c9fc (dataset), name: DatasetName { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_5d056c46-671e-47fd-ab76-2549bb8d565b\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 224395f7-7013-4e98-a10b-1a27fba2cb76 (dataset): DatasetConfig { id: 224395f7-7013-4e98-a10b-1a27fba2cb76 (dataset), name: DatasetName { pool_name: ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 255e40ed-72a0-49f2-b191-d2c58a101e7a (dataset): DatasetConfig { id: 255e40ed-72a0-49f2-b191-d2c58a101e7a (dataset), name: DatasetName { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 263da620-dfda-463f-81f5-9cb7a8d55d78 (dataset): DatasetConfig { id: 263da620-dfda-463f-81f5-9cb7a8d55d78 (dataset), name: DatasetName { pool_name: ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2bbbb532-6a35-4757-8026-112b24d29c8c (dataset): DatasetConfig { id: 2bbbb532-6a35-4757-8026-112b24d29c8c (dataset), name: DatasetName { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2d222798-1722-4032-9442-595b64fb9c72 (dataset): DatasetConfig { id: 2d222798-1722-4032-9442-595b64fb9c72 (dataset), name: DatasetName { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }, kind: TransientZone { name: \"oxz_ntp_5066b4aa-939d-40f6-9e2e-cc4b97bad1b4\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 30d07520-6505-4735-86d9-967c83566692 (dataset): DatasetConfig { id: 30d07520-6505-4735-86d9-967c83566692 (dataset), name: DatasetName { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 3501b99d-ebd0-4043-9d2b-2cdf765209a0 (dataset): DatasetConfig { id: 3501b99d-ebd0-4043-9d2b-2cdf765209a0 (dataset), name: DatasetName { pool_name: ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 3a8628c3-b3af-4711-bb33-b78ef591fc32 (dataset): DatasetConfig { id: 3a8628c3-b3af-4711-bb33-b78ef591fc32 (dataset), name: DatasetName { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 404e27c6-412e-4a54-b4cf-76c0f97b2511 (dataset): DatasetConfig { id: 404e27c6-412e-4a54-b4cf-76c0f97b2511 (dataset), name: DatasetName { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 410eda1d-afde-47e5-b024-9b1b296a7901 (dataset): DatasetConfig { id: 410eda1d-afde-47e5-b024-9b1b296a7901 (dataset), name: DatasetName { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 41d925fe-958a-46e8-8a49-308d37b7d2e8 (dataset): DatasetConfig { id: 41d925fe-958a-46e8-8a49-308d37b7d2e8 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }, kind: TransientZone { name: \"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 42418f47-5277-4c78-871f-402a3421b37e (dataset): DatasetConfig { id: 42418f47-5277-4c78-871f-402a3421b37e (dataset), name: DatasetName { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_3eae245b-f0e3-4234-b3b9-d9b4de089123\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4867bd09-4803-41ee-a90c-e4342f18006a (dataset): DatasetConfig { id: 4867bd09-4803-41ee-a90c-e4342f18006a (dataset), name: DatasetName { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4ffb9050-fdc0-4c7f-a051-24cfba157107 (dataset): DatasetConfig { id: 4ffb9050-fdc0-4c7f-a051-24cfba157107 (dataset), name: DatasetName { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_ddac3753-9787-478c-a9ea-d655291375e0\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 54d7cfe6-edc2-4ac9-817d-051437877429 (dataset): DatasetConfig { id: 54d7cfe6-edc2-4ac9-817d-051437877429 (dataset), name: DatasetName { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 59325252-4c21-4601-95e3-d5b1dce0b903 (dataset): DatasetConfig { id: 59325252-4c21-4601-95e3-d5b1dce0b903 (dataset), name: DatasetName { pool_name: ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 5f06e668-e204-4ce7-ae3f-d984d8ed1b72 (dataset): DatasetConfig { id: 5f06e668-e204-4ce7-ae3f-d984d8ed1b72 (dataset), name: DatasetName { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 618e79a3-9bff-407a-988d-1ef5542b6395 (dataset): DatasetConfig { id: 618e79a3-9bff-407a-988d-1ef5542b6395 (dataset), name: DatasetName { pool_name: ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 6567494d-6ab4-4c97-845f-2e925e3b4869 (dataset): DatasetConfig { id: 6567494d-6ab4-4c97-845f-2e925e3b4869 (dataset), name: DatasetName { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 73e54ba0-78a8-4b79-8268-f82178c0bd09 (dataset): DatasetConfig { id: 73e54ba0-78a8-4b79-8268-f82178c0bd09 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 75419486-7387-40ae-ab8b-c9e704811c0e (dataset): DatasetConfig { id: 75419486-7387-40ae-ab8b-c9e704811c0e (dataset), name: DatasetName { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 774c5501-9a1e-41b9-871b-4711fbb69b47 (dataset): DatasetConfig { id: 774c5501-9a1e-41b9-871b-4711fbb69b47 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 7dff74e3-fc83-426e-806a-05fd3a0ba17c (dataset): DatasetConfig { id: 7dff74e3-fc83-426e-806a-05fd3a0ba17c (dataset), name: DatasetName { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 81575a93-1176-49c5-abe6-74869028ae91 (dataset): DatasetConfig { id: 81575a93-1176-49c5-abe6-74869028ae91 (dataset), name: DatasetName { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 84903b24-098a-49c6-98c6-0a5f36d90c30 (dataset): DatasetConfig { id: 84903b24-098a-49c6-98c6-0a5f36d90c30 (dataset), name: DatasetName { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 8eb45ee7-de27-42c5-9e39-5e2718c27080 (dataset): DatasetConfig { id: 8eb45ee7-de27-42c5-9e39-5e2718c27080 (dataset), name: DatasetName { pool_name: ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 8fa52739-3f41-49fe-ab62-95cec230ed00 (dataset): DatasetConfig { id: 8fa52739-3f41-49fe-ab62-95cec230ed00 (dataset), name: DatasetName { pool_name: ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 923d43cb-7c6d-4a0b-be05-dae6b8fcfb72 (dataset): DatasetConfig { id: 923d43cb-7c6d-4a0b-be05-dae6b8fcfb72 (dataset), name: DatasetName { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }, kind: Clickhouse }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 95e77b7f-6e98-472b-860e-5fb33fc56184 (dataset): DatasetConfig { id: 95e77b7f-6e98-472b-860e-5fb33fc56184 (dataset), name: DatasetName { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_e9e770f2-88d4-4893-909a-79400af2db8b\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 9a023026-73b2-4510-b2c3-03248cce6a4b (dataset): DatasetConfig { id: 9a023026-73b2-4510-b2c3-03248cce6a4b (dataset), name: DatasetName { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 9d9f4dad-2b4a-4c5d-a51e-654497fc5702 (dataset): DatasetConfig { id: 9d9f4dad-2b4a-4c5d-a51e-654497fc5702 (dataset), name: DatasetName { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, a444f215-0327-43ec-b554-27e37f79bce8 (dataset): DatasetConfig { id: a444f215-0327-43ec-b554-27e37f79bce8 (dataset), name: DatasetName { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ac7115dd-6920-45f7-9567-7caed2e5f86f (dataset): DatasetConfig { id: ac7115dd-6920-45f7-9567-7caed2e5f86f (dataset), name: DatasetName { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, af5db44d-7812-4539-ba97-dd2fbb892e8d (dataset): DatasetConfig { id: af5db44d-7812-4539-ba97-dd2fbb892e8d (dataset), name: DatasetName { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c03b55e9-423c-4005-9892-160d9d5dd789 (dataset): DatasetConfig { id: c03b55e9-423c-4005-9892-160d9d5dd789 (dataset), name: DatasetName { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c801c24b-6251-487c-a10e-14268a365bda (dataset): DatasetConfig { id: c801c24b-6251-487c-a10e-14268a365bda (dataset), name: DatasetName { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_e786554c-6b77-4063-92f2-2aeb18d93f3d\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, cfb4124e-303f-4432-958d-d679486558e0 (dataset): DatasetConfig { id: cfb4124e-303f-4432-958d-d679486558e0 (dataset), name: DatasetName { pool_name: ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d181401b-2844-428a-b972-34677c9821dd (dataset): DatasetConfig { id: d181401b-2844-428a-b972-34677c9821dd (dataset), name: DatasetName { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d2f61512-9537-4ebb-83d9-6099f8ded8da (dataset): DatasetConfig { id: d2f61512-9537-4ebb-83d9-6099f8ded8da (dataset), name: DatasetName { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d397d7ee-da06-46bc-b2d7-ab3753e61534 (dataset): DatasetConfig { id: d397d7ee-da06-46bc-b2d7-ab3753e61534 (dataset), name: DatasetName { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, d5845d0f-7e70-49aa-a027-88659c18d08a (dataset): DatasetConfig { id: d5845d0f-7e70-49aa-a027-88659c18d08a (dataset), name: DatasetName { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, d7c7588c-80df-44cc-844e-e99894082f9f (dataset): DatasetConfig { id: d7c7588c-80df-44cc-844e-e99894082f9f (dataset), name: DatasetName { pool_name: ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d9adc725-c919-4e92-b909-a19cfa29fa35 (dataset): DatasetConfig { id: d9adc725-c919-4e92-b909-a19cfa29fa35 (dataset), name: DatasetName { pool_name: ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d9ba1a23-616c-4a8b-82a5-d8650cf6d09e (dataset): DatasetConfig { id: d9ba1a23-616c-4a8b-82a5-d8650cf6d09e (dataset), name: DatasetName { pool_name: ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, dac24b66-98af-4063-a49e-14df234d7e7b (dataset): DatasetConfig { id: dac24b66-98af-4063-a49e-14df234d7e7b (dataset), name: DatasetName { pool_name: ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, de5d2735-8d6c-4b29-9a9c-d042f3377f10 (dataset): DatasetConfig { id: de5d2735-8d6c-4b29-9a9c-d042f3377f10 (dataset), name: DatasetName { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e8ef6605-2026-449b-8461-e7da4642535d (dataset): DatasetConfig { id: e8ef6605-2026-449b-8461-e7da4642535d (dataset), name: DatasetName { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e9327d79-676c-45fe-9615-c303a7e76419 (dataset): DatasetConfig { id: e9327d79-676c-45fe-9615-c303a7e76419 (dataset), name: DatasetName { pool_name: ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, ed1cf8bd-0de7-49d3-af81-ca7404e62755 (dataset): DatasetConfig { id: ed1cf8bd-0de7-49d3-af81-ca7404e62755 (dataset), name: DatasetName { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }, kind: TransientZone { name: \"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f56613fc-74b7-4a92-9e7c-d8e0845db710 (dataset): DatasetConfig { id: f56613fc-74b7-4a92-9e7c-d8e0845db710 (dataset), name: DatasetName { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, fd1372a3-71b8-4d43-868b-c83f9b7bb746 (dataset): DatasetConfig { id: fd1372a3-71b8-4d43-868b-c83f9b7bb746 (dataset), name: DatasetName { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }, kind: TransientZone { name: \"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, fddcbc0b-f31d-4993-a152-7d781571ddce (dataset): DatasetConfig { id: fddcbc0b-f31d-4993-a152-7d781571ddce (dataset), name: DatasetName { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, fed79b14-392b-4b9a-a2a6-8023ff86abc4 (dataset): DatasetConfig { id: fed79b14-392b-4b9a-a2a6-8023ff86abc4 (dataset), name: DatasetName { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ff78fae3-cb12-4b99-a032-13380e44a8e5 (dataset): DatasetConfig { id: ff78fae3-cb12-4b99-a032-13380e44a8e5 (dataset), name: DatasetName { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }} }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:18.165445733Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:457"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:18.165475943Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:40335"} {"msg":"datasets ensure","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:18.166001287Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:849"} {"msg":"Received DatasetsEnsure { config: DatasetsConfig { generation: Generation(1), datasets: {03f95c58-edde-4ef2-b4e3-cc6750a67b6a (dataset): DatasetConfig { id: 03f95c58-edde-4ef2-b4e3-cc6750a67b6a (dataset), name: DatasetName { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 08d01d66-7e43-4073-b6d7-ea6917fea0a8 (dataset): DatasetConfig { id: 08d01d66-7e43-4073-b6d7-ea6917fea0a8 (dataset), name: DatasetName { pool_name: ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 098371c0-6e51-482e-a7d7-0162e882f407 (dataset): DatasetConfig { id: 098371c0-6e51-482e-a7d7-0162e882f407 (dataset), name: DatasetName { pool_name: ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 0cd5c4d9-d23c-49bd-8eac-8c52fafbc7ef (dataset): DatasetConfig { id: 0cd5c4d9-d23c-49bd-8eac-8c52fafbc7ef (dataset), name: DatasetName { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 0d738e08-d7ba-4c02-b6d7-1720dbda5ab8 (dataset): DatasetConfig { id: 0d738e08-d7ba-4c02-b6d7-1720dbda5ab8 (dataset), name: DatasetName { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_62e83095-0557-4ac0-8b12-7429e79e4bb6\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 14483828-0a7a-41c4-a00f-141ccc8c88b5 (dataset): DatasetConfig { id: 14483828-0a7a-41c4-a00f-141ccc8c88b5 (dataset), name: DatasetName { pool_name: ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }, kind: TransientZone { name: \"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 15039f4a-dcb6-44bc-a0b5-f12bb083a809 (dataset): DatasetConfig { id: 15039f4a-dcb6-44bc-a0b5-f12bb083a809 (dataset), name: DatasetName { pool_name: ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 15c7ae7f-8366-4e2d-873c-4cc3ee02a5fe (dataset): DatasetConfig { id: 15c7ae7f-8366-4e2d-873c-4cc3ee02a5fe (dataset), name: DatasetName { pool_name: ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 1ae1a305-21b6-43f9-a045-d321285817f7 (dataset): DatasetConfig { id: 1ae1a305-21b6-43f9-a045-d321285817f7 (dataset), name: DatasetName { pool_name: ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 2039e497-4e63-4c59-8e48-d06006699ec6 (dataset): DatasetConfig { id: 2039e497-4e63-4c59-8e48-d06006699ec6 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 207aaac7-e112-4299-b3eb-05f2e29a65e6 (dataset): DatasetConfig { id: 207aaac7-e112-4299-b3eb-05f2e29a65e6 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_4554ce4c-90f4-4fd2-93bd-086596238122\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 20c72dfc-63f3-46d8-bbbd-4f5a7772c9fc (dataset): DatasetConfig { id: 20c72dfc-63f3-46d8-bbbd-4f5a7772c9fc (dataset), name: DatasetName { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_5d056c46-671e-47fd-ab76-2549bb8d565b\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 224395f7-7013-4e98-a10b-1a27fba2cb76 (dataset): DatasetConfig { id: 224395f7-7013-4e98-a10b-1a27fba2cb76 (dataset), name: DatasetName { pool_name: ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 255e40ed-72a0-49f2-b191-d2c58a101e7a (dataset): DatasetConfig { id: 255e40ed-72a0-49f2-b191-d2c58a101e7a (dataset), name: DatasetName { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 263da620-dfda-463f-81f5-9cb7a8d55d78 (dataset): DatasetConfig { id: 263da620-dfda-463f-81f5-9cb7a8d55d78 (dataset), name: DatasetName { pool_name: ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2bbbb532-6a35-4757-8026-112b24d29c8c (dataset): DatasetConfig { id: 2bbbb532-6a35-4757-8026-112b24d29c8c (dataset), name: DatasetName { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 2d222798-1722-4032-9442-595b64fb9c72 (dataset): DatasetConfig { id: 2d222798-1722-4032-9442-595b64fb9c72 (dataset), name: DatasetName { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }, kind: TransientZone { name: \"oxz_ntp_5066b4aa-939d-40f6-9e2e-cc4b97bad1b4\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 30d07520-6505-4735-86d9-967c83566692 (dataset): DatasetConfig { id: 30d07520-6505-4735-86d9-967c83566692 (dataset), name: DatasetName { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 3501b99d-ebd0-4043-9d2b-2cdf765209a0 (dataset): DatasetConfig { id: 3501b99d-ebd0-4043-9d2b-2cdf765209a0 (dataset), name: DatasetName { pool_name: ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 3a8628c3-b3af-4711-bb33-b78ef591fc32 (dataset): DatasetConfig { id: 3a8628c3-b3af-4711-bb33-b78ef591fc32 (dataset), name: DatasetName { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 404e27c6-412e-4a54-b4cf-76c0f97b2511 (dataset): DatasetConfig { id: 404e27c6-412e-4a54-b4cf-76c0f97b2511 (dataset), name: DatasetName { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 410eda1d-afde-47e5-b024-9b1b296a7901 (dataset): DatasetConfig { id: 410eda1d-afde-47e5-b024-9b1b296a7901 (dataset), name: DatasetName { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 41d925fe-958a-46e8-8a49-308d37b7d2e8 (dataset): DatasetConfig { id: 41d925fe-958a-46e8-8a49-308d37b7d2e8 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }, kind: TransientZone { name: \"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 42418f47-5277-4c78-871f-402a3421b37e (dataset): DatasetConfig { id: 42418f47-5277-4c78-871f-402a3421b37e (dataset), name: DatasetName { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_3eae245b-f0e3-4234-b3b9-d9b4de089123\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4867bd09-4803-41ee-a90c-e4342f18006a (dataset): DatasetConfig { id: 4867bd09-4803-41ee-a90c-e4342f18006a (dataset), name: DatasetName { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 4ffb9050-fdc0-4c7f-a051-24cfba157107 (dataset): DatasetConfig { id: 4ffb9050-fdc0-4c7f-a051-24cfba157107 (dataset), name: DatasetName { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_ddac3753-9787-478c-a9ea-d655291375e0\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 54d7cfe6-edc2-4ac9-817d-051437877429 (dataset): DatasetConfig { id: 54d7cfe6-edc2-4ac9-817d-051437877429 (dataset), name: DatasetName { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 59325252-4c21-4601-95e3-d5b1dce0b903 (dataset): DatasetConfig { id: 59325252-4c21-4601-95e3-d5b1dce0b903 (dataset), name: DatasetName { pool_name: ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 5f06e668-e204-4ce7-ae3f-d984d8ed1b72 (dataset): DatasetConfig { id: 5f06e668-e204-4ce7-ae3f-d984d8ed1b72 (dataset), name: DatasetName { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 618e79a3-9bff-407a-988d-1ef5542b6395 (dataset): DatasetConfig { id: 618e79a3-9bff-407a-988d-1ef5542b6395 (dataset), name: DatasetName { pool_name: ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 6567494d-6ab4-4c97-845f-2e925e3b4869 (dataset): DatasetConfig { id: 6567494d-6ab4-4c97-845f-2e925e3b4869 (dataset), name: DatasetName { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 73e54ba0-78a8-4b79-8268-f82178c0bd09 (dataset): DatasetConfig { id: 73e54ba0-78a8-4b79-8268-f82178c0bd09 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }, kind: InternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 75419486-7387-40ae-ab8b-c9e704811c0e (dataset): DatasetConfig { id: 75419486-7387-40ae-ab8b-c9e704811c0e (dataset), name: DatasetName { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 774c5501-9a1e-41b9-871b-4711fbb69b47 (dataset): DatasetConfig { id: 774c5501-9a1e-41b9-871b-4711fbb69b47 (dataset), name: DatasetName { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 7dff74e3-fc83-426e-806a-05fd3a0ba17c (dataset): DatasetConfig { id: 7dff74e3-fc83-426e-806a-05fd3a0ba17c (dataset), name: DatasetName { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 81575a93-1176-49c5-abe6-74869028ae91 (dataset): DatasetConfig { id: 81575a93-1176-49c5-abe6-74869028ae91 (dataset), name: DatasetName { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 84903b24-098a-49c6-98c6-0a5f36d90c30 (dataset): DatasetConfig { id: 84903b24-098a-49c6-98c6-0a5f36d90c30 (dataset), name: DatasetName { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 8eb45ee7-de27-42c5-9e39-5e2718c27080 (dataset): DatasetConfig { id: 8eb45ee7-de27-42c5-9e39-5e2718c27080 (dataset), name: DatasetName { pool_name: ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 8fa52739-3f41-49fe-ab62-95cec230ed00 (dataset): DatasetConfig { id: 8fa52739-3f41-49fe-ab62-95cec230ed00 (dataset), name: DatasetName { pool_name: ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, 923d43cb-7c6d-4a0b-be05-dae6b8fcfb72 (dataset): DatasetConfig { id: 923d43cb-7c6d-4a0b-be05-dae6b8fcfb72 (dataset), name: DatasetName { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }, kind: Clickhouse }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 95e77b7f-6e98-472b-860e-5fb33fc56184 (dataset): DatasetConfig { id: 95e77b7f-6e98-472b-860e-5fb33fc56184 (dataset), name: DatasetName { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }, kind: TransientZone { name: \"oxz_internal_dns_e9e770f2-88d4-4893-909a-79400af2db8b\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 9a023026-73b2-4510-b2c3-03248cce6a4b (dataset): DatasetConfig { id: 9a023026-73b2-4510-b2c3-03248cce6a4b (dataset), name: DatasetName { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }, kind: TransientZone { name: \"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, 9d9f4dad-2b4a-4c5d-a51e-654497fc5702 (dataset): DatasetConfig { id: 9d9f4dad-2b4a-4c5d-a51e-654497fc5702 (dataset), name: DatasetName { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, a444f215-0327-43ec-b554-27e37f79bce8 (dataset): DatasetConfig { id: a444f215-0327-43ec-b554-27e37f79bce8 (dataset), name: DatasetName { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ac7115dd-6920-45f7-9567-7caed2e5f86f (dataset): DatasetConfig { id: ac7115dd-6920-45f7-9567-7caed2e5f86f (dataset), name: DatasetName { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, af5db44d-7812-4539-ba97-dd2fbb892e8d (dataset): DatasetConfig { id: af5db44d-7812-4539-ba97-dd2fbb892e8d (dataset), name: DatasetName { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c03b55e9-423c-4005-9892-160d9d5dd789 (dataset): DatasetConfig { id: c03b55e9-423c-4005-9892-160d9d5dd789 (dataset), name: DatasetName { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, c801c24b-6251-487c-a10e-14268a365bda (dataset): DatasetConfig { id: c801c24b-6251-487c-a10e-14268a365bda (dataset), name: DatasetName { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_e786554c-6b77-4063-92f2-2aeb18d93f3d\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, cfb4124e-303f-4432-958d-d679486558e0 (dataset): DatasetConfig { id: cfb4124e-303f-4432-958d-d679486558e0 (dataset), name: DatasetName { pool_name: ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d181401b-2844-428a-b972-34677c9821dd (dataset): DatasetConfig { id: d181401b-2844-428a-b972-34677c9821dd (dataset), name: DatasetName { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d2f61512-9537-4ebb-83d9-6099f8ded8da (dataset): DatasetConfig { id: d2f61512-9537-4ebb-83d9-6099f8ded8da (dataset), name: DatasetName { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d397d7ee-da06-46bc-b2d7-ab3753e61534 (dataset): DatasetConfig { id: d397d7ee-da06-46bc-b2d7-ab3753e61534 (dataset), name: DatasetName { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, d5845d0f-7e70-49aa-a027-88659c18d08a (dataset): DatasetConfig { id: d5845d0f-7e70-49aa-a027-88659c18d08a (dataset), name: DatasetName { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, d7c7588c-80df-44cc-844e-e99894082f9f (dataset): DatasetConfig { id: d7c7588c-80df-44cc-844e-e99894082f9f (dataset), name: DatasetName { pool_name: ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d9adc725-c919-4e92-b909-a19cfa29fa35 (dataset): DatasetConfig { id: d9adc725-c919-4e92-b909-a19cfa29fa35 (dataset), name: DatasetName { pool_name: ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }, kind: Crucible }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, d9ba1a23-616c-4a8b-82a5-d8650cf6d09e (dataset): DatasetConfig { id: d9ba1a23-616c-4a8b-82a5-d8650cf6d09e (dataset), name: DatasetName { pool_name: ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, dac24b66-98af-4063-a49e-14df234d7e7b (dataset): DatasetConfig { id: dac24b66-98af-4063-a49e-14df234d7e7b (dataset), name: DatasetName { pool_name: ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, de5d2735-8d6c-4b29-9a9c-d042f3377f10 (dataset): DatasetConfig { id: de5d2735-8d6c-4b29-9a9c-d042f3377f10 (dataset), name: DatasetName { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }, kind: Cockroach }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e8ef6605-2026-449b-8461-e7da4642535d (dataset): DatasetConfig { id: e8ef6605-2026-449b-8461-e7da4642535d (dataset), name: DatasetName { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }, kind: TransientZone { name: \"oxz_cockroachdb_6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, e9327d79-676c-45fe-9615-c303a7e76419 (dataset): DatasetConfig { id: e9327d79-676c-45fe-9615-c303a7e76419 (dataset), name: DatasetName { pool_name: ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }, kind: Debug }, inner: SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None } }, ed1cf8bd-0de7-49d3-af81-ca7404e62755 (dataset): DatasetConfig { id: ed1cf8bd-0de7-49d3-af81-ca7404e62755 (dataset), name: DatasetName { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }, kind: TransientZone { name: \"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, f56613fc-74b7-4a92-9e7c-d8e0845db710 (dataset): DatasetConfig { id: f56613fc-74b7-4a92-9e7c-d8e0845db710 (dataset), name: DatasetName { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }, kind: TransientZone { name: \"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, fd1372a3-71b8-4d43-868b-c83f9b7bb746 (dataset): DatasetConfig { id: fd1372a3-71b8-4d43-868b-c83f9b7bb746 (dataset), name: DatasetName { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }, kind: TransientZone { name: \"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121\" } }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, fddcbc0b-f31d-4993-a152-7d781571ddce (dataset): DatasetConfig { id: fddcbc0b-f31d-4993-a152-7d781571ddce (dataset), name: DatasetName { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }, kind: ExternalDns }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, fed79b14-392b-4b9a-a2a6-8023ff86abc4 (dataset): DatasetConfig { id: fed79b14-392b-4b9a-a2a6-8023ff86abc4 (dataset), name: DatasetName { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }, ff78fae3-cb12-4b99-a032-13380e44a8e5 (dataset): DatasetConfig { id: ff78fae3-cb12-4b99-a032-13380e44a8e5 (dataset), name: DatasetName { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }, kind: TransientZoneRoot }, inner: SharedDatasetConfig { compression: Off, quota: None, reservation: None } }} }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:18.168594926Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"No ledger in /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:18.168741037Z","hostname":"sock","pid":1574,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:189"} {"msg":"No ledger in /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:18.168766057Z","hostname":"sock","pid":1574,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:189"} {"msg":"No previously-stored 'requested datasets', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:18.168771987Z","hostname":"sock","pid":1574,"request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:906"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:18.168776857Z","hostname":"sock","pid":1574,"name":"oxp_473f0b4e-a714-496d-8d5d-665782986893/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:18.168780987Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_473f0b4e-a714-496d-8d5d-665782986893/crypt/internal_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:18.253781571Z","hostname":"sock","pid":1574,"name":"oxp_28b058a3-a4b5-473e-8580-f5c606b6d30b/crypt/zone/oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:18.253822422Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/28b058a3-a4b5-473e-8580-f5c606b6d30b/crypt/zone/oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\"), full_name: \"oxp_28b058a3-a4b5-473e-8580-f5c606b6d30b/crypt/zone/oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/79cbda5a-2e27-489e-945d-860d86dec819)","v":0,"name":"SledAgent","level":50,"time":"2024-11-18T20:58:18.280753742Z","hostname":"sock","pid":1574,"component":"ProbeManager","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:18.349273993Z","hostname":"sock","pid":1574,"name":"oxp_80f7cc50-8d94-4fb0-b296-446e3acf3366/crypt/zone/oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:18.349314244Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/80f7cc50-8d94-4fb0-b296-446e3acf3366/crypt/zone/oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\"), full_name: \"oxp_80f7cc50-8d94-4fb0-b296-446e3acf3366/crypt/zone/oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:18.445065998Z","hostname":"sock","pid":1574,"name":"oxp_2536a198-fc1d-4175-aab5-a62f1506f3e4/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:18.445113048Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_2536a198-fc1d-4175-aab5-a62f1506f3e4/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:18.530687317Z","hostname":"sock","pid":1574,"name":"oxp_473f0b4e-a714-496d-8d5d-665782986893/crypt/zone/oxz_internal_dns_62e83095-0557-4ac0-8b12-7429e79e4bb6","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:18.530730487Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/473f0b4e-a714-496d-8d5d-665782986893/crypt/zone/oxz_internal_dns_62e83095-0557-4ac0-8b12-7429e79e4bb6\"), full_name: \"oxp_473f0b4e-a714-496d-8d5d-665782986893/crypt/zone/oxz_internal_dns_62e83095-0557-4ac0-8b12-7429e79e4bb6\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:18.626231539Z","hostname":"sock","pid":1574,"name":"oxp_28b058a3-a4b5-473e-8580-f5c606b6d30b/crypt/zone/oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:18.626283099Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/28b058a3-a4b5-473e-8580-f5c606b6d30b/crypt/zone/oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\"), full_name: \"oxp_28b058a3-a4b5-473e-8580-f5c606b6d30b/crypt/zone/oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:18.724005968Z","hostname":"sock","pid":1574,"name":"oxp_28b058a3-a4b5-473e-8580-f5c606b6d30b/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:18.724053969Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_28b058a3-a4b5-473e-8580-f5c606b6d30b/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:18.809759308Z","hostname":"sock","pid":1574,"name":"oxp_feac0a3f-e4dc-4a16-9eeb-839f74ad9665/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:18.809804868Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/feac0a3f-e4dc-4a16-9eeb-839f74ad9665/crypt/zone\"), full_name: \"oxp_feac0a3f-e4dc-4a16-9eeb-839f74ad9665/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:18.875975602Z","hostname":"sock","pid":1574,"name":"oxp_f334d74e-134c-4206-a5ba-9a6ac6ea7954/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:18.876020822Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/f334d74e-134c-4206-a5ba-9a6ac6ea7954/crypt/debug\"), full_name: \"oxp_f334d74e-134c-4206-a5ba-9a6ac6ea7954/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:18.941026047Z","hostname":"sock","pid":1574,"name":"oxp_3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:18.941067777Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:19.027601783Z","hostname":"sock","pid":1574,"name":"oxp_3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63/crypt/zone/oxz_internal_dns_4554ce4c-90f4-4fd2-93bd-086596238122","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:19.027644783Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63/crypt/zone/oxz_internal_dns_4554ce4c-90f4-4fd2-93bd-086596238122\"), full_name: \"oxp_3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63/crypt/zone/oxz_internal_dns_4554ce4c-90f4-4fd2-93bd-086596238122\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:19.133676974Z","hostname":"sock","pid":1574,"name":"oxp_bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6/crypt/zone/oxz_cockroachdb_5d056c46-671e-47fd-ab76-2549bb8d565b","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:19.133723164Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6/crypt/zone/oxz_cockroachdb_5d056c46-671e-47fd-ab76-2549bb8d565b\"), full_name: \"oxp_bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6/crypt/zone/oxz_cockroachdb_5d056c46-671e-47fd-ab76-2549bb8d565b\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:19.224446401Z","hostname":"sock","pid":1574,"name":"oxp_80f7cc50-8d94-4fb0-b296-446e3acf3366/crypt/zone/oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:19.224489661Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/80f7cc50-8d94-4fb0-b296-446e3acf3366/crypt/zone/oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\"), full_name: \"oxp_80f7cc50-8d94-4fb0-b296-446e3acf3366/crypt/zone/oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/79cbda5a-2e27-489e-945d-860d86dec819)","v":0,"name":"SledAgent","level":50,"time":"2024-11-18T20:58:19.282813056Z","hostname":"sock","pid":1574,"component":"ProbeManager","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:19.320447437Z","hostname":"sock","pid":1574,"name":"oxp_dd78d2d6-0d80-45fe-9f73-e75ef7da86ee/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:19.320489477Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_dd78d2d6-0d80-45fe-9f73-e75ef7da86ee/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:19.403795168Z","hostname":"sock","pid":1574,"name":"oxp_f334d74e-134c-4206-a5ba-9a6ac6ea7954/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:19.403838089Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_f334d74e-134c-4206-a5ba-9a6ac6ea7954/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:19.487455742Z","hostname":"sock","pid":1574,"name":"oxp_473f0b4e-a714-496d-8d5d-665782986893/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:19.487497683Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_473f0b4e-a714-496d-8d5d-665782986893/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:19.571757631Z","hostname":"sock","pid":1574,"name":"oxp_473f0b4e-a714-496d-8d5d-665782986893/crypt/zone/oxz_ntp_5066b4aa-939d-40f6-9e2e-cc4b97bad1b4","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:19.571796511Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/473f0b4e-a714-496d-8d5d-665782986893/crypt/zone/oxz_ntp_5066b4aa-939d-40f6-9e2e-cc4b97bad1b4\"), full_name: \"oxp_473f0b4e-a714-496d-8d5d-665782986893/crypt/zone/oxz_ntp_5066b4aa-939d-40f6-9e2e-cc4b97bad1b4\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:19.666390617Z","hostname":"sock","pid":1574,"name":"oxp_bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:19.666436087Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6/crypt/internal_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:19.749750779Z","hostname":"sock","pid":1574,"name":"oxp_feac0a3f-e4dc-4a16-9eeb-839f74ad9665/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:19.749789439Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_feac0a3f-e4dc-4a16-9eeb-839f74ad9665/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:19.834208119Z","hostname":"sock","pid":1574,"name":"oxp_473f0b4e-a714-496d-8d5d-665782986893/crypt/zone/oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:19.834244889Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/473f0b4e-a714-496d-8d5d-665782986893/crypt/zone/oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\"), full_name: \"oxp_473f0b4e-a714-496d-8d5d-665782986893/crypt/zone/oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:19.931506574Z","hostname":"sock","pid":1574,"name":"oxp_2536a198-fc1d-4175-aab5-a62f1506f3e4/crypt/zone/oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:19.931546585Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/2536a198-fc1d-4175-aab5-a62f1506f3e4/crypt/zone/oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\"), full_name: \"oxp_2536a198-fc1d-4175-aab5-a62f1506f3e4/crypt/zone/oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:20.029403894Z","hostname":"sock","pid":1574,"name":"oxp_bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:20.029444574Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:20.114517839Z","hostname":"sock","pid":1574,"name":"oxp_3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63/crypt/zone/oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:20.114565809Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63/crypt/zone/oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\"), full_name: \"oxp_3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63/crypt/zone/oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:20.210871157Z","hostname":"sock","pid":1574,"name":"oxp_3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63/crypt/zone/oxz_cockroachdb_3eae245b-f0e3-4234-b3b9-d9b4de089123","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:20.210914338Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63/crypt/zone/oxz_cockroachdb_3eae245b-f0e3-4234-b3b9-d9b4de089123\"), full_name: \"oxp_3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63/crypt/zone/oxz_cockroachdb_3eae245b-f0e3-4234-b3b9-d9b4de089123\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/79cbda5a-2e27-489e-945d-860d86dec819)","v":0,"name":"SledAgent","level":50,"time":"2024-11-18T20:58:20.284794629Z","hostname":"sock","pid":1574,"component":"ProbeManager","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:20.306586181Z","hostname":"sock","pid":1574,"name":"oxp_3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:20.306614291Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:20.327954881Z","hostname":"sock","pid":1574,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:387","component":"producer-server","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/producer/src/lib.rs:387","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"22.410514838s"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:20.394436276Z","hostname":"sock","pid":1574,"name":"oxp_473f0b4e-a714-496d-8d5d-665782986893/crypt/zone/oxz_cockroachdb_ddac3753-9787-478c-a9ea-d655291375e0","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:20.394477937Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/473f0b4e-a714-496d-8d5d-665782986893/crypt/zone/oxz_cockroachdb_ddac3753-9787-478c-a9ea-d655291375e0\"), full_name: \"oxp_473f0b4e-a714-496d-8d5d-665782986893/crypt/zone/oxz_cockroachdb_ddac3753-9787-478c-a9ea-d655291375e0\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:20.486864006Z","hostname":"sock","pid":1574,"name":"oxp_dd78d2d6-0d80-45fe-9f73-e75ef7da86ee/crypt/zone/oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:20.486904106Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/dd78d2d6-0d80-45fe-9f73-e75ef7da86ee/crypt/zone/oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\"), full_name: \"oxp_dd78d2d6-0d80-45fe-9f73-e75ef7da86ee/crypt/zone/oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:20.583288385Z","hostname":"sock","pid":1574,"name":"oxp_f334d74e-134c-4206-a5ba-9a6ac6ea7954/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:20.583321565Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/f334d74e-134c-4206-a5ba-9a6ac6ea7954/crypt/zone\"), full_name: \"oxp_f334d74e-134c-4206-a5ba-9a6ac6ea7954/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:20.650797938Z","hostname":"sock","pid":1574,"name":"oxp_bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:20.650830779Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6/crypt/debug\"), full_name: \"oxp_bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:20.717586876Z","hostname":"sock","pid":1574,"name":"oxp_f334d74e-134c-4206-a5ba-9a6ac6ea7954/crypt/zone/oxz_crucible_948190c1-2519-4134-a120-8d6900d50730","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:20.717620387Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/f334d74e-134c-4206-a5ba-9a6ac6ea7954/crypt/zone/oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\"), full_name: \"oxp_f334d74e-134c-4206-a5ba-9a6ac6ea7954/crypt/zone/oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:20.815717208Z","hostname":"sock","pid":1574,"name":"oxp_bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:20.815755039Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:20.903624734Z","hostname":"sock","pid":1574,"name":"oxp_3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63/crypt/internal_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:20.903661604Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63/crypt/internal_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:20.992432036Z","hostname":"sock","pid":1574,"name":"oxp_2536a198-fc1d-4175-aab5-a62f1506f3e4/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:20.992466726Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_2536a198-fc1d-4175-aab5-a62f1506f3e4/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:21.086069955Z","hostname":"sock","pid":1574,"name":"oxp_3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:21.086105905Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63/crypt/zone\"), full_name: \"oxp_3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:21.153738299Z","hostname":"sock","pid":1574,"name":"oxp_bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:21.15378108Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6/crypt/zone\"), full_name: \"oxp_bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:21.218817705Z","hostname":"sock","pid":1574,"name":"oxp_473f0b4e-a714-496d-8d5d-665782986893/crypt/zone/oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:21.218851035Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/473f0b4e-a714-496d-8d5d-665782986893/crypt/zone/oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2\"), full_name: \"oxp_473f0b4e-a714-496d-8d5d-665782986893/crypt/zone/oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/79cbda5a-2e27-489e-945d-860d86dec819)","v":0,"name":"SledAgent","level":50,"time":"2024-11-18T20:58:21.28654704Z","hostname":"sock","pid":1574,"component":"ProbeManager","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:21.312622664Z","hostname":"sock","pid":1574,"name":"oxp_473f0b4e-a714-496d-8d5d-665782986893/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:21.312655385Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_473f0b4e-a714-496d-8d5d-665782986893/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:21.394553105Z","hostname":"sock","pid":1574,"name":"oxp_28b058a3-a4b5-473e-8580-f5c606b6d30b/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:21.394587516Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/28b058a3-a4b5-473e-8580-f5c606b6d30b/crypt/zone\"), full_name: \"oxp_28b058a3-a4b5-473e-8580-f5c606b6d30b/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:21.458444162Z","hostname":"sock","pid":1574,"name":"oxp_28b058a3-a4b5-473e-8580-f5c606b6d30b/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:21.458477462Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/28b058a3-a4b5-473e-8580-f5c606b6d30b/crypt/debug\"), full_name: \"oxp_28b058a3-a4b5-473e-8580-f5c606b6d30b/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:21.519465107Z","hostname":"sock","pid":1574,"name":"oxp_473f0b4e-a714-496d-8d5d-665782986893/crypt/clickhouse","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:21.519496857Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_473f0b4e-a714-496d-8d5d-665782986893/crypt/clickhouse\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:21.605131216Z","hostname":"sock","pid":1574,"name":"oxp_bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6/crypt/zone/oxz_internal_dns_e9e770f2-88d4-4893-909a-79400af2db8b","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:21.605176266Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6/crypt/zone/oxz_internal_dns_e9e770f2-88d4-4893-909a-79400af2db8b\"), full_name: \"oxp_bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6/crypt/zone/oxz_internal_dns_e9e770f2-88d4-4893-909a-79400af2db8b\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:21.698556063Z","hostname":"sock","pid":1574,"name":"oxp_bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6/crypt/zone/oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:21.698593673Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6/crypt/zone/oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\"), full_name: \"oxp_bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6/crypt/zone/oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:21.79481989Z","hostname":"sock","pid":1574,"name":"oxp_dd78d2d6-0d80-45fe-9f73-e75ef7da86ee/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:21.794866181Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/dd78d2d6-0d80-45fe-9f73-e75ef7da86ee/crypt/debug\"), full_name: \"oxp_dd78d2d6-0d80-45fe-9f73-e75ef7da86ee/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:21.859190371Z","hostname":"sock","pid":1574,"name":"oxp_473f0b4e-a714-496d-8d5d-665782986893/crypt/external_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:21.859227251Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_473f0b4e-a714-496d-8d5d-665782986893/crypt/external_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:21.947699801Z","hostname":"sock","pid":1574,"name":"oxp_2536a198-fc1d-4175-aab5-a62f1506f3e4/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:21.947737451Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/2536a198-fc1d-4175-aab5-a62f1506f3e4/crypt/debug\"), full_name: \"oxp_2536a198-fc1d-4175-aab5-a62f1506f3e4/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:22.012913217Z","hostname":"sock","pid":1574,"name":"oxp_bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6/crypt/zone/oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:22.012957137Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6/crypt/zone/oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139\"), full_name: \"oxp_bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6/crypt/zone/oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:22.108767042Z","hostname":"sock","pid":1574,"name":"oxp_bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6/crypt/zone/oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:22.108804632Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6/crypt/zone/oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3\"), full_name: \"oxp_bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6/crypt/zone/oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:22.209387652Z","hostname":"sock","pid":1574,"name":"oxp_2536a198-fc1d-4175-aab5-a62f1506f3e4/crypt/zone/oxz_cockroachdb_e786554c-6b77-4063-92f2-2aeb18d93f3d","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:22.209438353Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/2536a198-fc1d-4175-aab5-a62f1506f3e4/crypt/zone/oxz_cockroachdb_e786554c-6b77-4063-92f2-2aeb18d93f3d\"), full_name: \"oxp_2536a198-fc1d-4175-aab5-a62f1506f3e4/crypt/zone/oxz_cockroachdb_e786554c-6b77-4063-92f2-2aeb18d93f3d\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/79cbda5a-2e27-489e-945d-860d86dec819)","v":0,"name":"SledAgent","level":50,"time":"2024-11-18T20:58:22.289878203Z","hostname":"sock","pid":1574,"component":"ProbeManager","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:22.306511537Z","hostname":"sock","pid":1574,"name":"oxp_80f7cc50-8d94-4fb0-b296-446e3acf3366/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:22.306546167Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/80f7cc50-8d94-4fb0-b296-446e3acf3366/crypt/zone\"), full_name: \"oxp_80f7cc50-8d94-4fb0-b296-446e3acf3366/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:22.374554624Z","hostname":"sock","pid":1574,"name":"oxp_3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63/crypt/zone/oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:22.374594605Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63/crypt/zone/oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\"), full_name: \"oxp_3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63/crypt/zone/oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:22.491544297Z","hostname":"sock","pid":1574,"name":"oxp_473f0b4e-a714-496d-8d5d-665782986893/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:22.491594567Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/473f0b4e-a714-496d-8d5d-665782986893/crypt/zone\"), full_name: \"oxp_473f0b4e-a714-496d-8d5d-665782986893/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:22.563606364Z","hostname":"sock","pid":1574,"name":"oxp_473f0b4e-a714-496d-8d5d-665782986893/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:22.563640704Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/473f0b4e-a714-496d-8d5d-665782986893/crypt/debug\"), full_name: \"oxp_473f0b4e-a714-496d-8d5d-665782986893/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:22.631525161Z","hostname":"sock","pid":1574,"name":"oxp_3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:22.631558031Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63/crypt/debug\"), full_name: \"oxp_3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:22.697432342Z","hostname":"sock","pid":1574,"name":"oxp_f334d74e-134c-4206-a5ba-9a6ac6ea7954/crypt/zone/oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:22.697473943Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/f334d74e-134c-4206-a5ba-9a6ac6ea7954/crypt/zone/oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\"), full_name: \"oxp_f334d74e-134c-4206-a5ba-9a6ac6ea7954/crypt/zone/oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:22.795774046Z","hostname":"sock","pid":1574,"name":"oxp_80f7cc50-8d94-4fb0-b296-446e3acf3366/crucible","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:22.795828166Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_80f7cc50-8d94-4fb0-b296-446e3acf3366/crucible\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:22.882738315Z","hostname":"sock","pid":1574,"name":"oxp_feac0a3f-e4dc-4a16-9eeb-839f74ad9665/crypt/zone/oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:22.882776035Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/feac0a3f-e4dc-4a16-9eeb-839f74ad9665/crypt/zone/oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\"), full_name: \"oxp_feac0a3f-e4dc-4a16-9eeb-839f74ad9665/crypt/zone/oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:22.980677114Z","hostname":"sock","pid":1574,"name":"oxp_feac0a3f-e4dc-4a16-9eeb-839f74ad9665/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:22.980711505Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/feac0a3f-e4dc-4a16-9eeb-839f74ad9665/crypt/debug\"), full_name: \"oxp_feac0a3f-e4dc-4a16-9eeb-839f74ad9665/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.048262189Z","hostname":"sock","pid":1574,"name":"oxp_dd78d2d6-0d80-45fe-9f73-e75ef7da86ee/crypt/cockroachdb","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.048294809Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_dd78d2d6-0d80-45fe-9f73-e75ef7da86ee/crypt/cockroachdb\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.143089556Z","hostname":"sock","pid":1574,"name":"oxp_dd78d2d6-0d80-45fe-9f73-e75ef7da86ee/crypt/zone/oxz_cockroachdb_6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.143128306Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/dd78d2d6-0d80-45fe-9f73-e75ef7da86ee/crypt/zone/oxz_cockroachdb_6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce\"), full_name: \"oxp_dd78d2d6-0d80-45fe-9f73-e75ef7da86ee/crypt/zone/oxz_cockroachdb_6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.244674953Z","hostname":"sock","pid":1574,"name":"oxp_80f7cc50-8d94-4fb0-b296-446e3acf3366/crypt/debug","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.244721054Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/80f7cc50-8d94-4fb0-b296-446e3acf3366/crypt/debug\"), full_name: \"oxp_80f7cc50-8d94-4fb0-b296-446e3acf3366/crypt/debug\" }","config":"SharedDatasetConfig { compression: GzipN { level: GzipLevel(9) }, quota: Some(ByteCount(107374182400)), reservation: None }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/79cbda5a-2e27-489e-945d-860d86dec819)","v":0,"name":"SledAgent","level":50,"time":"2024-11-18T20:58:23.292143348Z","hostname":"sock","pid":1574,"component":"ProbeManager","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.314564945Z","hostname":"sock","pid":1574,"name":"oxp_473f0b4e-a714-496d-8d5d-665782986893/crypt/zone/oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.314600365Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/473f0b4e-a714-496d-8d5d-665782986893/crypt/zone/oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d\"), full_name: \"oxp_473f0b4e-a714-496d-8d5d-665782986893/crypt/zone/oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.415302916Z","hostname":"sock","pid":1574,"name":"oxp_dd78d2d6-0d80-45fe-9f73-e75ef7da86ee/crypt/zone/oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.415340926Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/dd78d2d6-0d80-45fe-9f73-e75ef7da86ee/crypt/zone/oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\"), full_name: \"oxp_dd78d2d6-0d80-45fe-9f73-e75ef7da86ee/crypt/zone/oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.516126798Z","hostname":"sock","pid":1574,"name":"oxp_473f0b4e-a714-496d-8d5d-665782986893/crypt/zone/oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.516164548Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/473f0b4e-a714-496d-8d5d-665782986893/crypt/zone/oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121\"), full_name: \"oxp_473f0b4e-a714-496d-8d5d-665782986893/crypt/zone/oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.577139723Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4345"} {"msg":"Ensuring there is a default route","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.577182603Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4355","gateway":"fd00:1122:3344:101::1"} {"msg":"Default route already exists","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.583091458Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4369","gateway":"fd00:1122:3344:101::1"} {"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.583124108Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4434"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.626928274Z","hostname":"sock","pid":1574,"name":"oxp_3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63/crypt/external_dns","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.626969055Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: true, mountpoint: Path(\"/data\"), full_name: \"oxp_3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63/crypt/external_dns\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.727408354Z","hostname":"sock","pid":1574,"name":"oxp_2536a198-fc1d-4175-aab5-a62f1506f3e4/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.727461864Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/2536a198-fc1d-4175-aab5-a62f1506f3e4/crypt/zone\"), full_name: \"oxp_2536a198-fc1d-4175-aab5-a62f1506f3e4/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"refreshed dendrite service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.729061496Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4500"} {"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.729104087Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4527"} {"msg":"refreshed lldpd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.773659319Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4540"} {"msg":"configuring MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.773688319Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4386"} {"msg":"Ensuring dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.804762221Z","hostname":"sock","pid":1574,"name":"oxp_dd78d2d6-0d80-45fe-9f73-e75ef7da86ee/crypt/zone","request":"datasets_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:952"} {"msg":"ensure_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.804866132Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:1338","details":"DatasetCreationDetails { zoned: false, mountpoint: Path(\"/pool/ext/dd78d2d6-0d80-45fe-9f73-e75ef7da86ee/crypt/zone\"), full_name: \"oxp_dd78d2d6-0d80-45fe-9f73-e75ef7da86ee/crypt/zone\" }","config":"SharedDatasetConfig { compression: Off, quota: None, reservation: None }"} {"msg":"refreshed MGS service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.821676137Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4428"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.821704937Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4507","rack_subnet":"fd00:1122:3344::"} {"msg":"refreshed wicketd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.840772699Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4518"} {"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.840801109Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4560"} {"msg":"Writing ledger to /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/.omicron-datasets.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.880731087Z","hostname":"sock","pid":1574,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:200"} {"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.881848516Z","hostname":"sock","pid":1574,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:133","path":"\"/pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-datasets.json\""} {"msg":"Writing ledger to /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/.omicron-datasets.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.881855976Z","hostname":"sock","pid":1574,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:200"} {"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.882086268Z","hostname":"sock","pid":1574,"request":"datasets_ensure","component":"StorageManager","file":"common/src/ledger.rs:133","path":"\"/pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-datasets.json\""} {"msg":"datasets ensure: Updated storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.882207498Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:851"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.88241945Z","hostname":"sock","pid":1574,"uri":"/datasets","method":"PUT","req_id":"fd44f69d-5d7d-4823-a62f-ede79e380321","remote_addr":"[fd00:1122:3344:101::1]:40335","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":5716807,"response_code":"200"} {"msg":"refreshed mgd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.907615428Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4592"} {"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.907639778Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4598"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: 62e83095-0557-4ac0-8b12-7429e79e4bb6 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4554ce4c-90f4-4fd2-93bd-086596238122 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: e9e770f2-88d4-4893-909a-79400af2db8b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.92802638Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.930958382Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:64692"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.931571366Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"No ledger in /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.931647917Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:189"} {"msg":"No ledger in /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.931655027Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:189"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.931743498Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.931750518Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.931938949Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.93207184Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.932171321Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.983652915Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.983679895Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.983685475Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"refreshed mg-ddm service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.997206406Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4639"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:23.997249196Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:370"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_62e83095-0557-4ac0-8b12-7429e79e4bb6","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:24.037738168Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_4554ce4c-90f4-4fd2-93bd-086596238122","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:24.041300145Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_e9e770f2-88d4-4893-909a-79400af2db8b","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:24.044576349Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_internal_dns_62e83095-0557-4ac0-8b12-7429e79e4bb6","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:24.056393357Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:24.060569788Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:418"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/79cbda5a-2e27-489e-945d-860d86dec819)","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:24.06076826Z","hostname":"sock","pid":1574,"component":"NexusNotifierTask","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/nexus.rs:527"} {"msg":"Installing Omicron zone: oxz_internal_dns_4554ce4c-90f4-4fd2-93bd-086596238122","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:24.060944021Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_internal_dns_e9e770f2-88d4-4893-909a-79400af2db8b","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:24.065017722Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:24.098861324Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:686"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:24.098962105Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:731"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:24.099479019Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:734","uptime":"1"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:24.099516409Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:440","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.2.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: Some(\n None,\n ),\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n tx_eq: None,\n },\n },\n },\n}"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:24.145427571Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:24.207163732Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:24.207229062Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:24.207240192Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:24.207248262Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:24.207256062Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:24.207262522Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:24.207266882Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:24.207273433Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:24.207281193Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:24.207287713Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:24.207293413Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:24.207299823Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:24.207315913Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:24.207322343Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:24.207328103Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:24.207334053Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:24.207338493Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:24.207344143Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:24.207349773Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:24.207359853Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:24.207366073Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:24.207372463Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:24.207378123Z","hostname":"sock","pid":1574,"component":"StorageResources","file":"sled-storage/src/resources.rs:550","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-18T20:58:24.242981699Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13668364515274527681"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-18T20:58:24.24314791Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5937130538126233288"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/79cbda5a-2e27-489e-945d-860d86dec819)","v":0,"name":"SledAgent","level":50,"time":"2024-11-18T20:58:24.293325564Z","hostname":"sock","pid":1574,"component":"ProbeManager","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-18T20:58:24.762424553Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12123728031068596224"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-18T20:58:24.762526194Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"11959324033292328996"} {"msg":"Profile for oxz_internal_dns_e9e770f2-88d4-4893-909a-79400af2db8b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:25.686490355Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:25.686589815Z","hostname":"sock","pid":1574,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:3::, width: 64 }"} {"msg":"Booting oxz_internal_dns_e9e770f2-88d4-4893-909a-79400af2db8b zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:25.686751497Z","hostname":"sock","pid":1574,"zone":"oxz_internal_dns_e9e770f2-88d4-4893-909a-79400af2db8b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_internal_dns_62e83095-0557-4ac0-8b12-7429e79e4bb6:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:26.173377416Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:26.653629788Z","hostname":"sock","pid":1574,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:1::, width: 64 }"} {"msg":"Profile for oxz_internal_dns_4554ce4c-90f4-4fd2-93bd-086596238122:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:26.653790149Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:26.653806329Z","hostname":"sock","pid":1574,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:2::, width: 64 }"} {"msg":"wait for service Some(\"oxz_internal_dns_e9e770f2-88d4-4893-909a-79400af2db8b\") failed: Property not found. retry in 58.732598ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:26.662312132Z","hostname":"sock","pid":1574,"zone":"oxz_internal_dns_e9e770f2-88d4-4893-909a-79400af2db8b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_internal_dns_62e83095-0557-4ac0-8b12-7429e79e4bb6 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:26.662404643Z","hostname":"sock","pid":1574,"zone":"oxz_internal_dns_62e83095-0557-4ac0-8b12-7429e79e4bb6","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_internal_dns_4554ce4c-90f4-4fd2-93bd-086596238122 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:26.665836389Z","hostname":"sock","pid":1574,"zone":"oxz_internal_dns_4554ce4c-90f4-4fd2-93bd-086596238122","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_internal_dns_e9e770f2-88d4-4893-909a-79400af2db8b\") failed: Property not found. retry in 89.953384ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:26.732935069Z","hostname":"sock","pid":1574,"zone":"oxz_internal_dns_e9e770f2-88d4-4893-909a-79400af2db8b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_62e83095-0557-4ac0-8b12-7429e79e4bb6\") failed: Property not found. retry in 25.955528ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:26.793490741Z","hostname":"sock","pid":1574,"zone":"oxz_internal_dns_62e83095-0557-4ac0-8b12-7429e79e4bb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_4554ce4c-90f4-4fd2-93bd-086596238122\") failed: Property not found. retry in 34.393913ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:26.801144218Z","hostname":"sock","pid":1574,"zone":"oxz_internal_dns_4554ce4c-90f4-4fd2-93bd-086596238122","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_62e83095-0557-4ac0-8b12-7429e79e4bb6\") failed: Property not found. retry in 96.369596ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:26.827060311Z","hostname":"sock","pid":1574,"zone":"oxz_internal_dns_62e83095-0557-4ac0-8b12-7429e79e4bb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_e9e770f2-88d4-4893-909a-79400af2db8b\") failed: Property not found. retry in 222.547978ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:26.834833629Z","hostname":"sock","pid":1574,"zone":"oxz_internal_dns_e9e770f2-88d4-4893-909a-79400af2db8b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_4554ce4c-90f4-4fd2-93bd-086596238122\") failed: Property not found. retry in 94.938482ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:26.844686773Z","hostname":"sock","pid":1574,"zone":"oxz_internal_dns_4554ce4c-90f4-4fd2-93bd-086596238122","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_62e83095-0557-4ac0-8b12-7429e79e4bb6\") failed: Property not found. retry in 152.074692ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:26.976189183Z","hostname":"sock","pid":1574,"zone":"oxz_internal_dns_62e83095-0557-4ac0-8b12-7429e79e4bb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_4554ce4c-90f4-4fd2-93bd-086596238122\") failed: Property not found. retry in 119.300651ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:26.995529467Z","hostname":"sock","pid":1574,"zone":"oxz_internal_dns_4554ce4c-90f4-4fd2-93bd-086596238122","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_e9e770f2-88d4-4893-909a-79400af2db8b\") failed: Property not found. retry in 281.186115ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:27.106569805Z","hostname":"sock","pid":1574,"zone":"oxz_internal_dns_e9e770f2-88d4-4893-909a-79400af2db8b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_4554ce4c-90f4-4fd2-93bd-086596238122\") failed: Property not found. retry in 585.443723ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:27.139979294Z","hostname":"sock","pid":1574,"zone":"oxz_internal_dns_4554ce4c-90f4-4fd2-93bd-086596238122","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_62e83095-0557-4ac0-8b12-7429e79e4bb6\") failed: Property not found. retry in 235.494011ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:27.157438465Z","hostname":"sock","pid":1574,"zone":"oxz_internal_dns_62e83095-0557-4ac0-8b12-7429e79e4bb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_e9e770f2-88d4-4893-909a-79400af2db8b\") failed: Property not found. retry in 1.135700971s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:27.397362534Z","hostname":"sock","pid":1574,"zone":"oxz_internal_dns_e9e770f2-88d4-4893-909a-79400af2db8b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_62e83095-0557-4ac0-8b12-7429e79e4bb6\") failed: Property not found. retry in 671.752564ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:27.404991231Z","hostname":"sock","pid":1574,"zone":"oxz_internal_dns_62e83095-0557-4ac0-8b12-7429e79e4bb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_4554ce4c-90f4-4fd2-93bd-086596238122\") failed: Property not found. retry in 1.043321444s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:27.735403746Z","hostname":"sock","pid":1574,"zone":"oxz_internal_dns_4554ce4c-90f4-4fd2-93bd-086596238122","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_62e83095-0557-4ac0-8b12-7429e79e4bb6\") failed: Property not found. retry in 828.877537ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:28.086383633Z","hostname":"sock","pid":1574,"zone":"oxz_internal_dns_62e83095-0557-4ac0-8b12-7429e79e4bb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_e9e770f2-88d4-4893-909a-79400af2db8b\") failed: Property not found. retry in 1.38198412s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:28.542124852Z","hostname":"sock","pid":1574,"zone":"oxz_internal_dns_e9e770f2-88d4-4893-909a-79400af2db8b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_4554ce4c-90f4-4fd2-93bd-086596238122\") failed: Property not found. retry in 1.077842475s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:28.787398821Z","hostname":"sock","pid":1574,"zone":"oxz_internal_dns_4554ce4c-90f4-4fd2-93bd-086596238122","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_62e83095-0557-4ac0-8b12-7429e79e4bb6\") failed: Property not found. retry in 986.140898ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:28.923903259Z","hostname":"sock","pid":1574,"zone":"oxz_internal_dns_62e83095-0557-4ac0-8b12-7429e79e4bb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_4554ce4c-90f4-4fd2-93bd-086596238122\") failed: Property not found. retry in 809.382992ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:29.875530867Z","hostname":"sock","pid":1574,"zone":"oxz_internal_dns_4554ce4c-90f4-4fd2-93bd-086596238122","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_62e83095-0557-4ac0-8b12-7429e79e4bb6\") failed: Property not found. retry in 926.476149ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:29.919208382Z","hostname":"sock","pid":1574,"zone":"oxz_internal_dns_62e83095-0557-4ac0-8b12-7429e79e4bb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_e9e770f2-88d4-4893-909a-79400af2db8b\") failed: Property not found. retry in 781.633891ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:29.932110059Z","hostname":"sock","pid":1574,"zone":"oxz_internal_dns_e9e770f2-88d4-4893-909a-79400af2db8b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_4554ce4c-90f4-4fd2-93bd-086596238122\") failed: Property not found. retry in 923.765529ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:30.694274043Z","hostname":"sock","pid":1574,"zone":"oxz_internal_dns_4554ce4c-90f4-4fd2-93bd-086596238122","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_e9e770f2-88d4-4893-909a-79400af2db8b\") failed: Property not found. retry in 1.285759919s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:30.724225996Z","hostname":"sock","pid":1574,"zone":"oxz_internal_dns_e9e770f2-88d4-4893-909a-79400af2db8b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_62e83095-0557-4ac0-8b12-7429e79e4bb6\") failed: Property not found. retry in 1.12246954s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:30.855148773Z","hostname":"sock","pid":1574,"zone":"oxz_internal_dns_62e83095-0557-4ac0-8b12-7429e79e4bb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_4554ce4c-90f4-4fd2-93bd-086596238122\") failed: Property not found. retry in 1.31114288s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:31.627354761Z","hostname":"sock","pid":1574,"zone":"oxz_internal_dns_4554ce4c-90f4-4fd2-93bd-086596238122","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_62e83095-0557-4ac0-8b12-7429e79e4bb6\") failed: Property not found. retry in 569.354154ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:31.987384017Z","hostname":"sock","pid":1574,"zone":"oxz_internal_dns_62e83095-0557-4ac0-8b12-7429e79e4bb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_e9e770f2-88d4-4893-909a-79400af2db8b\") failed: Property not found. retry in 1.427639257s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:32.018307857Z","hostname":"sock","pid":1574,"zone":"oxz_internal_dns_e9e770f2-88d4-4893-909a-79400af2db8b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_62e83095-0557-4ac0-8b12-7429e79e4bb6\") failed: Property not found. retry in 809.353788ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:32.565984172Z","hostname":"sock","pid":1574,"zone":"oxz_internal_dns_62e83095-0557-4ac0-8b12-7429e79e4bb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_4554ce4c-90f4-4fd2-93bd-086596238122\") failed: Property not found. retry in 768.98349ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:32.947358196Z","hostname":"sock","pid":1574,"zone":"oxz_internal_dns_4554ce4c-90f4-4fd2-93bd-086596238122","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_62e83095-0557-4ac0-8b12-7429e79e4bb6\") failed: Property not found. retry in 1.276270285s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:33.384624447Z","hostname":"sock","pid":1574,"zone":"oxz_internal_dns_62e83095-0557-4ac0-8b12-7429e79e4bb6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_e9e770f2-88d4-4893-909a-79400af2db8b\") failed: Property not found. retry in 580.708435ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:33.454606929Z","hostname":"sock","pid":1574,"zone":"oxz_internal_dns_e9e770f2-88d4-4893-909a-79400af2db8b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_4554ce4c-90f4-4fd2-93bd-086596238122\") failed: Property not found. retry in 1.487453708s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:33.72552843Z","hostname":"sock","pid":1574,"zone":"oxz_internal_dns_4554ce4c-90f4-4fd2-93bd-086596238122","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-18T20:58:34.244777402Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13668364515274527681"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-18T20:58:34.244889553Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5937130538126233288"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-18T20:58:34.764581159Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12123728031068596224"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-18T20:58:34.765145943Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"11959324033292328996"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:35.231978785Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3317","zone":"oxz_internal_dns_e9e770f2-88d4-4893-909a-79400af2db8b"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:35.232023775Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3317","zone":"oxz_internal_dns_62e83095-0557-4ac0-8b12-7429e79e4bb6"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:35.232028715Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3317","zone":"oxz_internal_dns_4554ce4c-90f4-4fd2-93bd-086596238122"} {"msg":"Writing ledger to /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:35.232034475Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:200"} {"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:35.232371998Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:133","path":"\"/pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-zones.json\""} {"msg":"Writing ledger to /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:35.232387658Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:200"} {"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:35.23263876Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:133","path":"\"/pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-zones.json\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:35.232744291Z","hostname":"sock","pid":1574,"uri":"/omicron-zones","method":"PUT","req_id":"7175c5fc-7e60-4df4-b4af-bea9e285cb39","remote_addr":"[fd00:1122:3344:101::1]:64692","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":11301439,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:35.234649235Z","hostname":"sock","pid":1574,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:38.719620456Z","hostname":"sock","pid":1574,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:38.798703485Z","hostname":"sock","pid":1574,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:38.882253439Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:691"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:38.882278429Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:38.915325155Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:38.916133191Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:38.916157541Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: 62e83095-0557-4ac0-8b12-7429e79e4bb6 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4554ce4c-90f4-4fd2-93bd-086596238122 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: e9e770f2-88d4-4893-909a-79400af2db8b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b99c98c4-f6a4-43cc-a789-4b40c33f469f, kind: Service { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 }, name: Name(\"ntp-5066b4aa-939d-40f6-9e2e-cc4b97bad1b4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 130, 96])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:38.945657801Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:38.945792453Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:37676"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:38.946041255Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:38.946061515Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:38.946149175Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:38.946195005Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:38.946204176Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:38.946325706Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:38.946449678Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:38.946495968Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:38.959227453Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:38.990292784Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:38.990998879Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:38.991011179Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:39.02061441Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 130, 96]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:39.020651831Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:1271","zone_type":"boundary_ntp"} {"msg":"no nat entry found for: 10.151.2.169","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:39.020962373Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.169","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:39.020967503Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.2.169","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:39.021213905Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:39.021596558Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Configuring new Omicron zone: oxz_ntp_5066b4aa-939d-40f6-9e2e-cc4b97bad1b4","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:39.045746268Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_ntp_5066b4aa-939d-40f6-9e2e-cc4b97bad1b4","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:39.064500798Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Profile for oxz_ntp_5066b4aa-939d-40f6-9e2e-cc4b97bad1b4:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:39.661572191Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_ntp_5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:39.661637571Z","hostname":"sock","pid":1574,"zone":"oxz_ntp_5066b4aa-939d-40f6-9e2e-cc4b97bad1b4","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_ntp_5066b4aa-939d-40f6-9e2e-cc4b97bad1b4\") failed: Property not found. retry in 43.985677ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:39.799656961Z","hostname":"sock","pid":1574,"zone":"oxz_ntp_5066b4aa-939d-40f6-9e2e-cc4b97bad1b4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_5066b4aa-939d-40f6-9e2e-cc4b97bad1b4\") failed: Property not found. retry in 140.395974ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:39.851407267Z","hostname":"sock","pid":1574,"zone":"oxz_ntp_5066b4aa-939d-40f6-9e2e-cc4b97bad1b4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_5066b4aa-939d-40f6-9e2e-cc4b97bad1b4\") failed: Property not found. retry in 163.965675ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:40.00341304Z","hostname":"sock","pid":1574,"zone":"oxz_ntp_5066b4aa-939d-40f6-9e2e-cc4b97bad1b4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_5066b4aa-939d-40f6-9e2e-cc4b97bad1b4\") failed: Property not found. retry in 301.888478ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:40.177190046Z","hostname":"sock","pid":1574,"zone":"oxz_ntp_5066b4aa-939d-40f6-9e2e-cc4b97bad1b4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_5066b4aa-939d-40f6-9e2e-cc4b97bad1b4\") failed: Property not found. retry in 510.493416ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:40.488021194Z","hostname":"sock","pid":1574,"zone":"oxz_ntp_5066b4aa-939d-40f6-9e2e-cc4b97bad1b4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/79cbda5a-2e27-489e-945d-860d86dec819)","v":0,"name":"SledAgent","level":50,"time":"2024-11-18T20:58:40.697680158Z","hostname":"sock","pid":1574,"component":"ProbeManager","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:231"} {"msg":"wait for service Some(\"oxz_ntp_5066b4aa-939d-40f6-9e2e-cc4b97bad1b4\") failed: Property not found. retry in 874.075482ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:41.008152573Z","hostname":"sock","pid":1574,"zone":"oxz_ntp_5066b4aa-939d-40f6-9e2e-cc4b97bad1b4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/79cbda5a-2e27-489e-945d-860d86dec819)","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:41.064538004Z","hostname":"sock","pid":1574,"component":"NexusNotifierTask","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/nexus.rs:527"} {"msg":"wait for service Some(\"oxz_ntp_5066b4aa-939d-40f6-9e2e-cc4b97bad1b4\") failed: Property not found. retry in 1.14142064s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:41.892050496Z","hostname":"sock","pid":1574,"zone":"oxz_ntp_5066b4aa-939d-40f6-9e2e-cc4b97bad1b4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_5066b4aa-939d-40f6-9e2e-cc4b97bad1b4\") failed: Property not found. retry in 993.9782ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:43.042935709Z","hostname":"sock","pid":1574,"zone":"oxz_ntp_5066b4aa-939d-40f6-9e2e-cc4b97bad1b4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_5066b4aa-939d-40f6-9e2e-cc4b97bad1b4\") failed: Property not found. retry in 1.312867573s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:44.045460506Z","hostname":"sock","pid":1574,"zone":"oxz_ntp_5066b4aa-939d-40f6-9e2e-cc4b97bad1b4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-18T20:58:44.057950009Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10791944641750664615"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-18T20:58:44.246349914Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13668364515274527681"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-18T20:58:44.246584816Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5937130538126233288"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-18T20:58:44.686508677Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5084187938249036381"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-18T20:58:44.766622865Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12123728031068596224"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-18T20:58:44.766820106Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"11959324033292328996"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-18T20:58:45.236014535Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"11342567266855366024"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:45.378487637Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3317","zone":"oxz_ntp_5066b4aa-939d-40f6-9e2e-cc4b97bad1b4"} {"msg":"Writing ledger to /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:45.378555208Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:200"} {"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:45.37888413Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:133","path":"\"/pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-zones.json\""} {"msg":"Writing ledger to /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:45.378906211Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:200"} {"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:45.379139382Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:133","path":"\"/pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-zones.json\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:45.379240753Z","hostname":"sock","pid":1574,"uri":"/omicron-zones","method":"PUT","req_id":"56d0c30c-6c54-4a37-8d82-2841d883f5bc","remote_addr":"[fd00:1122:3344:101::1]:37676","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":6433332,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:45.381230268Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:732"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:45.410710778Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:45.410860899Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:53592"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_5066b4aa-939d-40f6-9e2e-cc4b97bad1b4': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-11-18T20:58:45.420103668Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3953"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:45.420201928Z","hostname":"sock","pid":1574,"uri":"/timesync","method":"GET","req_id":"edf7bd3b-180c-47c1-bf18-fc78b7f6cf84","remote_addr":"[fd00:1122:3344:101::1]:53592","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":9104,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:45.422402765Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:45.550332909Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:45.55048043Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:44058"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_5066b4aa-939d-40f6-9e2e-cc4b97bad1b4': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-11-18T20:58:49.558709734Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3953"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:49.558821665Z","hostname":"sock","pid":1574,"uri":"/timesync","method":"GET","req_id":"96e8e462-5ddc-47e6-8e9e-fa19afe9464a","remote_addr":"[fd00:1122:3344:101::1]:44058","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4008239,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:49.561084662Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:49.736935273Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:49.737107884Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:46659"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_5066b4aa-939d-40f6-9e2e-cc4b97bad1b4': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-11-18T20:58:49.745580347Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3953"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:49.745678598Z","hostname":"sock","pid":1574,"uri":"/timesync","method":"GET","req_id":"63e0f00f-1416-4d7a-a8fb-b2aee8668340","remote_addr":"[fd00:1122:3344:101::1]:46659","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8482,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:49.74725502Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:49.994484493Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:49.994596584Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:59101"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_5066b4aa-939d-40f6-9e2e-cc4b97bad1b4': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-11-18T20:58:50.003171458Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3953"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:50.003266929Z","hostname":"sock","pid":1574,"uri":"/timesync","method":"GET","req_id":"32a77f85-970b-41e6-87af-a4040ff2af82","remote_addr":"[fd00:1122:3344:101::1]:59101","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8531,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:50.005502865Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:50.170507476Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:50.170618317Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:61499"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:53.78583593Z","hostname":"sock","pid":1574,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:32981"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:53.786032491Z","hostname":"sock","pid":1574,"uri":"/baseboard","method":"GET","req_id":"663608a1-4815-496b-b01a-9f3dda0e6aa7","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:32981","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":82,"response_code":"200"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-18T20:58:54.059877643Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"10791944641750664615"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:54.180243011Z","hostname":"sock","pid":1574,"uri":"/timesync","method":"GET","req_id":"2ef2986b-8a03-42cd-8f05-341a6bf9372c","remote_addr":"[fd00:1122:3344:101::1]:61499","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":4009555,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:54.18276013Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:740"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:54.18277228Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-18T20:58:54.249332416Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"13668364515274527681"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-18T20:58:54.249465357Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5937130538126233288"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:54.511484342Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:54.511617572Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:37874"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:54.520227467Z","hostname":"sock","pid":1574,"uri":"/timesync","method":"GET","req_id":"21b140a7-afad-4543-a6e7-c9fc0552bf80","remote_addr":"[fd00:1122:3344:101::1]:37874","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":8464,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:54.523180549Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:740"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:54.523193299Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:54.702202304Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:54.702374895Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:62540"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-18T20:58:54.706574126Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"5084187938249036381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:54.710430815Z","hostname":"sock","pid":1574,"uri":"/timesync","method":"GET","req_id":"61b1c235-388a-49c6-a623-60ba358c4040","remote_addr":"[fd00:1122:3344:101::1]:62540","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7950,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:54.712160318Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:740"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:54.712170138Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:762","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-18T20:58:54.768287277Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"12123728031068596224"} {"msg":"failed to sample kstat target, requeueing","v":0,"name":"SledAgent","level":50,"time":"2024-11-18T20:58:54.768406818Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:496","error":"NoSuchKstat","id":"11959324033292328996"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:55.196053492Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:712"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:55.196194763Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:50644"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:55.216381174Z","hostname":"sock","pid":1574,"uri":"/timesync","method":"GET","req_id":"b7461b99-c6a1-4469-9f56-b5538d11f11b","remote_addr":"[fd00:1122:3344:101::1]:50644","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":20048,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 398108675, ip_addr: 23.186.168.3, stratum: 3, ref_time: 1731963534.2237487, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:55.218880473Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:740"} {"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:55.218886313Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1425"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 62e83095-0557-4ac0-8b12-7429e79e4bb6 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4554ce4c-90f4-4fd2-93bd-086596238122 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: e9e770f2-88d4-4893-909a-79400af2db8b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b99c98c4-f6a4-43cc-a789-4b40c33f469f, kind: Service { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 }, name: Name(\"ntp-5066b4aa-939d-40f6-9e2e-cc4b97bad1b4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 130, 96])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ddac3753-9787-478c-a9ea-d655291375e0 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3eae245b-f0e3-4234-b3b9-d9b4de089123 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5d056c46-671e-47fd-ab76-2549bb8d565b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: e786554c-6b77-4063-92f2-2aeb18d93f3d (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:55.248269952Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:55.248382923Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:33135"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:55.248793986Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:55.248873026Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:55.248982667Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:55.249076668Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:55.249105908Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:55.249250659Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:55.24936529Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:55.24941661Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:55.334265253Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:55.334336264Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:55.334348184Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:55.334357534Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:55.334366094Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_ddac3753-9787-478c-a9ea-d655291375e0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:55.423487499Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_5d056c46-671e-47fd-ab76-2549bb8d565b","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:55.427442008Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_3eae245b-f0e3-4234-b3b9-d9b4de089123","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:55.43167289Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_e786554c-6b77-4063-92f2-2aeb18d93f3d","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:55.435653639Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:55.439932791Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_cockroachdb_ddac3753-9787-478c-a9ea-d655291375e0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:55.444240344Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_cockroachdb_5d056c46-671e-47fd-ab76-2549bb8d565b","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:55.448470285Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_cockroachdb_3eae245b-f0e3-4234-b3b9-d9b4de089123","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:55.452912388Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_cockroachdb_e786554c-6b77-4063-92f2-2aeb18d93f3d","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:55.458100877Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_cockroachdb_6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:55.463364156Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:57.764124696Z","hostname":"sock","pid":1574,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"163.444169ms"} {"msg":"Profile for oxz_cockroachdb_ddac3753-9787-478c-a9ea-d655291375e0:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:58.30016271Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_cockroachdb_5d056c46-671e-47fd-ab76-2549bb8d565b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:58.300217994Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_ddac3753-9787-478c-a9ea-d655291375e0 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:58.300298911Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_ddac3753-9787-478c-a9ea-d655291375e0","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_cockroachdb_5d056c46-671e-47fd-ab76-2549bb8d565b zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:58.305783172Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_5d056c46-671e-47fd-ab76-2549bb8d565b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_e786554c-6b77-4063-92f2-2aeb18d93f3d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:58.314021174Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_e786554c-6b77-4063-92f2-2aeb18d93f3d zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:58.314194457Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_e786554c-6b77-4063-92f2-2aeb18d93f3d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:58.31888422Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:58.319044661Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_cockroachdb_3eae245b-f0e3-4234-b3b9-d9b4de089123:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:58.341321768Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_3eae245b-f0e3-4234-b3b9-d9b4de089123 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:58:58.341503449Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_3eae245b-f0e3-4234-b3b9-d9b4de089123","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_cockroachdb_ddac3753-9787-478c-a9ea-d655291375e0\") failed: Property not found. retry in 41.418656ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:58.446645786Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_ddac3753-9787-478c-a9ea-d655291375e0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e786554c-6b77-4063-92f2-2aeb18d93f3d\") failed: Property not found. retry in 51.633529ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:58.4539414Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_e786554c-6b77-4063-92f2-2aeb18d93f3d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5d056c46-671e-47fd-ab76-2549bb8d565b\") failed: Property not found. retry in 70.332823ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:58.460503779Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_5d056c46-671e-47fd-ab76-2549bb8d565b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce\") failed: Property not found. retry in 40.32756ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:58.468310358Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3eae245b-f0e3-4234-b3b9-d9b4de089123\") failed: Property not found. retry in 61.797501ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:58.476482179Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_3eae245b-f0e3-4234-b3b9-d9b4de089123","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ddac3753-9787-478c-a9ea-d655291375e0\") failed: Property not found. retry in 86.979759ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:58.496152116Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_ddac3753-9787-478c-a9ea-d655291375e0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e786554c-6b77-4063-92f2-2aeb18d93f3d\") failed: Property not found. retry in 54.094461ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:58.650649851Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_e786554c-6b77-4063-92f2-2aeb18d93f3d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ddac3753-9787-478c-a9ea-d655291375e0\") failed: Property not found. retry in 248.300382ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:59.368881292Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_ddac3753-9787-478c-a9ea-d655291375e0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5d056c46-671e-47fd-ab76-2549bb8d565b\") failed: Property not found. retry in 124.564126ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:59.669520266Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_5d056c46-671e-47fd-ab76-2549bb8d565b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3eae245b-f0e3-4234-b3b9-d9b4de089123\") failed: Property not found. retry in 66.212932ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:59.786813508Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_3eae245b-f0e3-4234-b3b9-d9b4de089123","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce\") failed: Property not found. retry in 80.219691ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:59.806283943Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ddac3753-9787-478c-a9ea-d655291375e0\") failed: Property not found. retry in 243.161298ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:59.820841932Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_ddac3753-9787-478c-a9ea-d655291375e0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5d056c46-671e-47fd-ab76-2549bb8d565b\") failed: Property not found. retry in 224.40972ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:59.836282668Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_5d056c46-671e-47fd-ab76-2549bb8d565b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e786554c-6b77-4063-92f2-2aeb18d93f3d\") failed: Property not found. retry in 294.336567ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:59.854181972Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_e786554c-6b77-4063-92f2-2aeb18d93f3d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3eae245b-f0e3-4234-b3b9-d9b4de089123\") failed: Property not found. retry in 209.184747ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:59.869502256Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_3eae245b-f0e3-4234-b3b9-d9b4de089123","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce\") failed: Property not found. retry in 144.295525ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:58:59.905478325Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce\") failed: Property not found. retry in 399.51097ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:00.064838757Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ddac3753-9787-478c-a9ea-d655291375e0\") failed: Property not found. retry in 785.873558ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:00.078138276Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_ddac3753-9787-478c-a9ea-d655291375e0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5d056c46-671e-47fd-ab76-2549bb8d565b\") failed: Property not found. retry in 579.136258ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:00.091453746Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_5d056c46-671e-47fd-ab76-2549bb8d565b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3eae245b-f0e3-4234-b3b9-d9b4de089123\") failed: Property not found. retry in 397.913277ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:00.105519231Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_3eae245b-f0e3-4234-b3b9-d9b4de089123","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e786554c-6b77-4063-92f2-2aeb18d93f3d\") failed: Property not found. retry in 386.046402ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:00.177178497Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_e786554c-6b77-4063-92f2-2aeb18d93f3d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce\") failed: Property not found. retry in 479.446793ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:00.473920676Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3eae245b-f0e3-4234-b3b9-d9b4de089123\") failed: Property not found. retry in 973.110842ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:00.529069759Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_3eae245b-f0e3-4234-b3b9-d9b4de089123","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e786554c-6b77-4063-92f2-2aeb18d93f3d\") failed: Property not found. retry in 976.53906ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:00.577221539Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_e786554c-6b77-4063-92f2-2aeb18d93f3d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5d056c46-671e-47fd-ab76-2549bb8d565b\") failed: Property not found. retry in 754.570606ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:00.683986677Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_5d056c46-671e-47fd-ab76-2549bb8d565b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ddac3753-9787-478c-a9ea-d655291375e0\") failed: Property not found. retry in 848.095189ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:00.875903632Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_ddac3753-9787-478c-a9ea-d655291375e0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce\") failed: Property not found. retry in 980.41244ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:00.967732439Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5d056c46-671e-47fd-ab76-2549bb8d565b\") failed: Property not found. retry in 1.330214343s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:01.466803931Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_5d056c46-671e-47fd-ab76-2549bb8d565b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3eae245b-f0e3-4234-b3b9-d9b4de089123\") failed: Property not found. retry in 664.878679ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:01.510263016Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_3eae245b-f0e3-4234-b3b9-d9b4de089123","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e786554c-6b77-4063-92f2-2aeb18d93f3d\") failed: Property not found. retry in 1.364486204s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:01.561035346Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_e786554c-6b77-4063-92f2-2aeb18d93f3d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ddac3753-9787-478c-a9ea-d655291375e0\") failed: Property not found. retry in 939.652363ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:01.731888783Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_ddac3753-9787-478c-a9ea-d655291375e0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce\") failed: Property not found. retry in 682.67923ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:01.955242233Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3eae245b-f0e3-4234-b3b9-d9b4de089123\") failed: Property not found. retry in 654.685214ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:02.182847096Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_3eae245b-f0e3-4234-b3b9-d9b4de089123","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce\") failed: Property not found. retry in 1.48649011s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:02.645828278Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ddac3753-9787-478c-a9ea-d655291375e0\") failed: Property not found. retry in 1.107428279s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:02.678248391Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_ddac3753-9787-478c-a9ea-d655291375e0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5d056c46-671e-47fd-ab76-2549bb8d565b\") failed: Property not found. retry in 804.382796ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:02.804531285Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_5d056c46-671e-47fd-ab76-2549bb8d565b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3eae245b-f0e3-4234-b3b9-d9b4de089123\") failed: Property not found. retry in 746.834205ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:02.844830417Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_3eae245b-f0e3-4234-b3b9-d9b4de089123","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e786554c-6b77-4063-92f2-2aeb18d93f3d\") failed: Property not found. retry in 1.288871409s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:02.931386084Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_e786554c-6b77-4063-92f2-2aeb18d93f3d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3eae245b-f0e3-4234-b3b9-d9b4de089123\") failed: Property not found. retry in 723.791574ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:03.600971651Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_3eae245b-f0e3-4234-b3b9-d9b4de089123","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5d056c46-671e-47fd-ab76-2549bb8d565b\") failed: Property not found. retry in 1.41730148s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:03.615681321Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_5d056c46-671e-47fd-ab76-2549bb8d565b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ddac3753-9787-478c-a9ea-d655291375e0\") failed: Property not found. retry in 677.180324ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:03.793292359Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_ddac3753-9787-478c-a9ea-d655291375e0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce\") failed: Property not found. retry in 564.700174ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:04.137809995Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e786554c-6b77-4063-92f2-2aeb18d93f3d\") failed: Property not found. retry in 721.391752ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:04.225929284Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_e786554c-6b77-4063-92f2-2aeb18d93f3d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3eae245b-f0e3-4234-b3b9-d9b4de089123\") failed: Property not found. retry in 600.023301ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:04.33236219Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_3eae245b-f0e3-4234-b3b9-d9b4de089123","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ddac3753-9787-478c-a9ea-d655291375e0\") failed: Property not found. retry in 826.002156ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:04.477004112Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_ddac3753-9787-478c-a9ea-d655291375e0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce\") failed: Property not found. retry in 1.090179432s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:04.708701925Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3eae245b-f0e3-4234-b3b9-d9b4de089123\") failed: Property not found. retry in 644.484369ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:04.939597192Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_3eae245b-f0e3-4234-b3b9-d9b4de089123","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e786554c-6b77-4063-92f2-2aeb18d93f3d\") failed: Property not found. retry in 1.490030248s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:04.952654309Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_e786554c-6b77-4063-92f2-2aeb18d93f3d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5d056c46-671e-47fd-ab76-2549bb8d565b\") failed: Property not found. retry in 795.307614ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:05.037116981Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_5d056c46-671e-47fd-ab76-2549bb8d565b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ddac3753-9787-478c-a9ea-d655291375e0\") failed: Property not found. retry in 804.363621ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:05.308638112Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_ddac3753-9787-478c-a9ea-d655291375e0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3eae245b-f0e3-4234-b3b9-d9b4de089123\") failed: Property not found. retry in 1.300915299s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:05.590973073Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_3eae245b-f0e3-4234-b3b9-d9b4de089123","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce\") failed: Property not found. retry in 901.171788ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:05.805502508Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5d056c46-671e-47fd-ab76-2549bb8d565b\") failed: Property not found. retry in 1.452089816s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:05.839038929Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_5d056c46-671e-47fd-ab76-2549bb8d565b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_ddac3753-9787-478c-a9ea-d655291375e0\") failed: Property not found. retry in 1.496436509s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:06.120373402Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_ddac3753-9787-478c-a9ea-d655291375e0","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e786554c-6b77-4063-92f2-2aeb18d93f3d\") failed: Property not found. retry in 641.978639ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:06.449326842Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_e786554c-6b77-4063-92f2-2aeb18d93f3d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce\") failed: Property not found. retry in 798.995501ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:06.713511708Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3eae245b-f0e3-4234-b3b9-d9b4de089123\") failed: Property not found. retry in 729.503207ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:06.897392683Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_3eae245b-f0e3-4234-b3b9-d9b4de089123","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e786554c-6b77-4063-92f2-2aeb18d93f3d\") failed: Property not found. retry in 987.49766ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:07.098609648Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_e786554c-6b77-4063-92f2-2aeb18d93f3d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_5d056c46-671e-47fd-ab76-2549bb8d565b\") failed: Property not found. retry in 1.342827182s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:07.29818682Z","hostname":"sock","pid":1574,"zone":"oxz_cockroachdb_5d056c46-671e-47fd-ab76-2549bb8d565b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:08.661080862Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3317","zone":"oxz_cockroachdb_ddac3753-9787-478c-a9ea-d655291375e0"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:08.661134212Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3317","zone":"oxz_cockroachdb_5d056c46-671e-47fd-ab76-2549bb8d565b"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:08.661142472Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3317","zone":"oxz_cockroachdb_3eae245b-f0e3-4234-b3b9-d9b4de089123"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:08.661148332Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3317","zone":"oxz_cockroachdb_e786554c-6b77-4063-92f2-2aeb18d93f3d"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:08.661155022Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3317","zone":"oxz_cockroachdb_6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce"} {"msg":"Writing ledger to /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:08.661173213Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:200"} {"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:08.661625666Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:133","path":"\"/pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-zones.json\""} {"msg":"Writing ledger to /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:08.661646126Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:200"} {"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:08.661918618Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:133","path":"\"/pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-zones.json\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:08.662014149Z","hostname":"sock","pid":1574,"uri":"/omicron-zones","method":"PUT","req_id":"ace9a1c0-bd93-408f-9187-dfda4c29fa94","remote_addr":"[fd00:1122:3344:101::1]:33135","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":13469368,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:08.696068714Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:51469"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:08.72635472Z","hostname":"sock","pid":1574,"uri":"/cockroachdb","method":"POST","req_id":"1f3a0f37-4e48-4c17-a7ea-36435c8d0ae5","remote_addr":"[fd00:1122:3344:101::1]:51469","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to get address: Failed to get address for name oxControlService7/omicron6 in oxz_cockroachdb_3eae245b-f0e3-4234-b3b9-d9b4de089123: Zone execution error: Command [/usr/sbin/zlogin oxz_cockroachdb_3eae245b-f0e3-4234-b3b9-d9b4de089123 /usr/sbin/ipadm show-addr -p -o ADDR oxControlService7/omicron6] executed and failed with status: exit status: 1 stdout: \n stderr: ipadm: Could not get address: Object not found","latency_us":30196,"response_code":"500"} {"msg":"Failed to initialize CockroachDB","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:08.726616852Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1154","retry_after":"64.237051ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1f3a0f37-4e48-4c17-a7ea-36435c8d0ae5\", \"content-length\": \"124\", \"date\": \"Mon, 18 Nov 2024 20:59:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1f3a0f37-4e48-4c17-a7ea-36435c8d0ae5\" }"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::4]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:08.824058871Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3796"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:20.534228941Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3814"} {"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:24.481349063Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3837"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:24.481873332Z","hostname":"sock","pid":1574,"uri":"/cockroachdb","method":"POST","req_id":"90989b30-1fdc-4da9-a5cc-e3b2dc100b99","remote_addr":"[fd00:1122:3344:101::1]:51469","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":15732600,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:24.482441766Z","hostname":"sock","pid":1574,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:61748"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:24.483692136Z","hostname":"sock","pid":1574,"uri":"/baseboard","method":"GET","req_id":"5450a222-94b0-4e91-963a-e18d0a168486","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:61748","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":96,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:24.489992683Z","hostname":"sock","pid":1574,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"671.393233ms"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 62e83095-0557-4ac0-8b12-7429e79e4bb6 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4554ce4c-90f4-4fd2-93bd-086596238122 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: e9e770f2-88d4-4893-909a-79400af2db8b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b99c98c4-f6a4-43cc-a789-4b40c33f469f, kind: Service { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 }, name: Name(\"ntp-5066b4aa-939d-40f6-9e2e-cc4b97bad1b4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 130, 96])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ddac3753-9787-478c-a9ea-d655291375e0 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3eae245b-f0e3-4234-b3b9-d9b4de089123 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5d056c46-671e-47fd-ab76-2549bb8d565b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: e786554c-6b77-4063-92f2-2aeb18d93f3d (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 14e637a2-51ce-4cac-b831-85b757e5c55d (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 95dce781-7d6b-4bf1-88f1-880b60d0adba, kind: Service { id: 14e637a2-51ce-4cac-b831-85b757e5c55d }, name: Name(\"external-dns-14e637a2-51ce-4cac-b831-85b757e5c55d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 238])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3b3dea8a-e84f-4acf-bafc-3b0e4a815dc5, kind: Service { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 }, name: Name(\"external-dns-be35e232-dd9b-4d59-b4e1-eadb7f69bb77\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 109])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: db0c0ea5-0d8f-4f9d-9979-f3989ce962ea, kind: Service { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 }, name: Name(\"nexus-16a7178a-dbb6-4392-b395-7bddb3817d13\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9b2d2174-b236-4546-bace-300e05afe3c2 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 5dbd0a76-35d9-4b29-8800-824968d1caba, kind: Service { id: 9b2d2174-b236-4546-bace-300e05afe3c2 }, name: Name(\"nexus-9b2d2174-b236-4546-bace-300e05afe3c2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 121])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e67bea26-1a67-4166-875f-3cab431644e6, kind: Service { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 }, name: Name(\"nexus-3dce42ed-74cb-4c8b-9326-8273a1efd563\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 239])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 84b5a094-9eb0-4ff8-be58-5c4913059740 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e5181def-6535-4350-b7f4-acf3785cf121 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b77d62e-b462-4293-950d-00a565b722bd (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 40c1b925-76fc-473c-9317-ffcfbf8aef2d (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 154903a9-f4ab-4352-b947-96c5fce154b3 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b5ceaf3-96d0-4f36-882b-965f9acc31c4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: c7378503-c5b8-4eee-91c0-2c15e3708aea (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 090b4de4-d9d1-47aa-96a9-4cc472900139 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: abc9c059-37e5-408c-b8e0-9fdb2f94fcfb (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ced4eec8-4bf4-46b3-843c-91242defe60a (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 08aaade2-f75a-444c-b97a-b4d3bf97b095 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External } } } }, OmicronZoneConfig { id: 04a81970-7129-40c7-b6f5-36219a8ad854 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cdeb736-9e69-46f9-afb0-59255807dbe4 (service), filesystem_pool: Some(ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External } } } }, OmicronZoneConfig { id: 948190c1-2519-4134-a120-8d6900d50730 (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:24.52444633Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:24.524946054Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::1]:49408"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:24.525855781Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:24.525913972Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:24.526123693Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:24.526260374Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:24.526392435Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:24.526763018Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:24.527447953Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:24.527521564Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:24.636029415Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:24.636087905Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:24.636104505Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:24.636110745Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:24.636114896Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:24.636120176Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:24.636126996Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:24.636132076Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:24.636137626Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:24.636143046Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:24.636149586Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:24.636154806Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:24.636160156Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:24.636165536Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:24.636170656Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:24.636177466Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:24.636183196Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:24.636202226Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:24.636207536Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:24.636307997Z","hostname":"sock","pid":1574,"uri":"/omicron-zones","method":"PUT","req_id":"495a0c85-3632-4826-b51a-d8c3f5e4c456","remote_addr":"[fd00:1122:3344:101::1]:49408","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121\", TimeNotSynchronized), (\"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3\", TimeNotSynchronized), (\"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\", TimeNotSynchronized), (\"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\", TimeNotSynchronized), (\"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2\", TimeNotSynchronized), (\"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\", TimeNotSynchronized), (\"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\", TimeNotSynchronized), (\"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\", TimeNotSynchronized), (\"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\", TimeNotSynchronized), (\"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\", TimeNotSynchronized), (\"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\", TimeNotSynchronized), (\"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\", TimeNotSynchronized), (\"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\", TimeNotSynchronized), (\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\", TimeNotSynchronized), (\"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d\", TimeNotSynchronized), (\"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\", TimeNotSynchronized), (\"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\", TimeNotSynchronized), (\"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\", TimeNotSynchronized), (\"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139\", TimeNotSynchronized)]","latency_us":111332,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:24.636487058Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"79.491355ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"495a0c85-3632-4826-b51a-d8c3f5e4c456\", \"content-length\": \"124\", \"date\": \"Mon, 18 Nov 2024 20:59:24 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"495a0c85-3632-4826-b51a-d8c3f5e4c456\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 62e83095-0557-4ac0-8b12-7429e79e4bb6 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4554ce4c-90f4-4fd2-93bd-086596238122 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: e9e770f2-88d4-4893-909a-79400af2db8b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b99c98c4-f6a4-43cc-a789-4b40c33f469f, kind: Service { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 }, name: Name(\"ntp-5066b4aa-939d-40f6-9e2e-cc4b97bad1b4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 130, 96])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ddac3753-9787-478c-a9ea-d655291375e0 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3eae245b-f0e3-4234-b3b9-d9b4de089123 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5d056c46-671e-47fd-ab76-2549bb8d565b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: e786554c-6b77-4063-92f2-2aeb18d93f3d (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 14e637a2-51ce-4cac-b831-85b757e5c55d (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 95dce781-7d6b-4bf1-88f1-880b60d0adba, kind: Service { id: 14e637a2-51ce-4cac-b831-85b757e5c55d }, name: Name(\"external-dns-14e637a2-51ce-4cac-b831-85b757e5c55d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 238])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3b3dea8a-e84f-4acf-bafc-3b0e4a815dc5, kind: Service { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 }, name: Name(\"external-dns-be35e232-dd9b-4d59-b4e1-eadb7f69bb77\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 109])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: db0c0ea5-0d8f-4f9d-9979-f3989ce962ea, kind: Service { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 }, name: Name(\"nexus-16a7178a-dbb6-4392-b395-7bddb3817d13\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9b2d2174-b236-4546-bace-300e05afe3c2 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 5dbd0a76-35d9-4b29-8800-824968d1caba, kind: Service { id: 9b2d2174-b236-4546-bace-300e05afe3c2 }, name: Name(\"nexus-9b2d2174-b236-4546-bace-300e05afe3c2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 121])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e67bea26-1a67-4166-875f-3cab431644e6, kind: Service { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 }, name: Name(\"nexus-3dce42ed-74cb-4c8b-9326-8273a1efd563\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 239])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 84b5a094-9eb0-4ff8-be58-5c4913059740 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e5181def-6535-4350-b7f4-acf3785cf121 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b77d62e-b462-4293-950d-00a565b722bd (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 40c1b925-76fc-473c-9317-ffcfbf8aef2d (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 154903a9-f4ab-4352-b947-96c5fce154b3 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b5ceaf3-96d0-4f36-882b-965f9acc31c4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: c7378503-c5b8-4eee-91c0-2c15e3708aea (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 090b4de4-d9d1-47aa-96a9-4cc472900139 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: abc9c059-37e5-408c-b8e0-9fdb2f94fcfb (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ced4eec8-4bf4-46b3-843c-91242defe60a (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 08aaade2-f75a-444c-b97a-b4d3bf97b095 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External } } } }, OmicronZoneConfig { id: 04a81970-7129-40c7-b6f5-36219a8ad854 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cdeb736-9e69-46f9-afb0-59255807dbe4 (service), filesystem_pool: Some(ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External } } } }, OmicronZoneConfig { id: 948190c1-2519-4134-a120-8d6900d50730 (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:24.716519587Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:24.716837669Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:24.716875209Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:24.716998281Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:24.717107771Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:24.717122581Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:24.717281983Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:24.717444954Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:24.717514434Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:24.820838657Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:24.820905487Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:24.820913507Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:24.820919078Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:24.820924188Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:24.820931128Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:24.820936268Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:24.820941848Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:24.820947248Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:24.820961768Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:24.820967418Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:24.820972848Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:24.820978068Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:24.820983768Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:24.820989138Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:24.820994328Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:24.821001188Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:24.821006438Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:24.821011688Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:24.821099399Z","hostname":"sock","pid":1574,"uri":"/omicron-zones","method":"PUT","req_id":"5d610ffa-eaee-4b31-89ba-4a3a8be4fb79","remote_addr":"[fd00:1122:3344:101::1]:49408","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\", TimeNotSynchronized), (\"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\", TimeNotSynchronized), (\"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3\", TimeNotSynchronized), (\"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\", TimeNotSynchronized), (\"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d\", TimeNotSynchronized), (\"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\", TimeNotSynchronized), (\"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139\", TimeNotSynchronized), (\"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\", TimeNotSynchronized), (\"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\", TimeNotSynchronized), (\"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\", TimeNotSynchronized), (\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\", TimeNotSynchronized), (\"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\", TimeNotSynchronized), (\"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\", TimeNotSynchronized), (\"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2\", TimeNotSynchronized), (\"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\", TimeNotSynchronized), (\"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121\", TimeNotSynchronized), (\"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\", TimeNotSynchronized), (\"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\", TimeNotSynchronized), (\"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\", TimeNotSynchronized)]","latency_us":104714,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:24.82128769Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"168.274979ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5d610ffa-eaee-4b31-89ba-4a3a8be4fb79\", \"content-length\": \"124\", \"date\": \"Mon, 18 Nov 2024 20:59:24 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5d610ffa-eaee-4b31-89ba-4a3a8be4fb79\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 62e83095-0557-4ac0-8b12-7429e79e4bb6 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4554ce4c-90f4-4fd2-93bd-086596238122 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: e9e770f2-88d4-4893-909a-79400af2db8b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b99c98c4-f6a4-43cc-a789-4b40c33f469f, kind: Service { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 }, name: Name(\"ntp-5066b4aa-939d-40f6-9e2e-cc4b97bad1b4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 130, 96])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ddac3753-9787-478c-a9ea-d655291375e0 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3eae245b-f0e3-4234-b3b9-d9b4de089123 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5d056c46-671e-47fd-ab76-2549bb8d565b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: e786554c-6b77-4063-92f2-2aeb18d93f3d (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 14e637a2-51ce-4cac-b831-85b757e5c55d (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 95dce781-7d6b-4bf1-88f1-880b60d0adba, kind: Service { id: 14e637a2-51ce-4cac-b831-85b757e5c55d }, name: Name(\"external-dns-14e637a2-51ce-4cac-b831-85b757e5c55d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 238])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3b3dea8a-e84f-4acf-bafc-3b0e4a815dc5, kind: Service { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 }, name: Name(\"external-dns-be35e232-dd9b-4d59-b4e1-eadb7f69bb77\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 109])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: db0c0ea5-0d8f-4f9d-9979-f3989ce962ea, kind: Service { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 }, name: Name(\"nexus-16a7178a-dbb6-4392-b395-7bddb3817d13\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9b2d2174-b236-4546-bace-300e05afe3c2 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 5dbd0a76-35d9-4b29-8800-824968d1caba, kind: Service { id: 9b2d2174-b236-4546-bace-300e05afe3c2 }, name: Name(\"nexus-9b2d2174-b236-4546-bace-300e05afe3c2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 121])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e67bea26-1a67-4166-875f-3cab431644e6, kind: Service { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 }, name: Name(\"nexus-3dce42ed-74cb-4c8b-9326-8273a1efd563\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 239])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 84b5a094-9eb0-4ff8-be58-5c4913059740 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e5181def-6535-4350-b7f4-acf3785cf121 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b77d62e-b462-4293-950d-00a565b722bd (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 40c1b925-76fc-473c-9317-ffcfbf8aef2d (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 154903a9-f4ab-4352-b947-96c5fce154b3 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b5ceaf3-96d0-4f36-882b-965f9acc31c4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: c7378503-c5b8-4eee-91c0-2c15e3708aea (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 090b4de4-d9d1-47aa-96a9-4cc472900139 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: abc9c059-37e5-408c-b8e0-9fdb2f94fcfb (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ced4eec8-4bf4-46b3-843c-91242defe60a (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 08aaade2-f75a-444c-b97a-b4d3bf97b095 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External } } } }, OmicronZoneConfig { id: 04a81970-7129-40c7-b6f5-36219a8ad854 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cdeb736-9e69-46f9-afb0-59255807dbe4 (service), filesystem_pool: Some(ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External } } } }, OmicronZoneConfig { id: 948190c1-2519-4134-a120-8d6900d50730 (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:24.990298414Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:24.990579496Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:24.990631627Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:24.990751537Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:24.990872348Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:24.990885518Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:24.991025239Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:24.99114696Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:24.991212741Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.09403516Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.09410325Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.094124921Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.09413101Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.094138181Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.09414395Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.094149461Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.094154951Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.094160341Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.094167251Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.094171291Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.094177971Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.094183081Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.094188301Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.094193481Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.094200351Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.094205551Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.094212431Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.094217841Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:25.094285532Z","hostname":"sock","pid":1574,"uri":"/omicron-zones","method":"PUT","req_id":"3a720b85-9fad-498b-8b55-af5171475da6","remote_addr":"[fd00:1122:3344:101::1]:49408","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\", TimeNotSynchronized), (\"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\", TimeNotSynchronized), (\"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\", TimeNotSynchronized), (\"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2\", TimeNotSynchronized), (\"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121\", TimeNotSynchronized), (\"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3\", TimeNotSynchronized), (\"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\", TimeNotSynchronized), (\"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\", TimeNotSynchronized), (\"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\", TimeNotSynchronized), (\"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\", TimeNotSynchronized), (\"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\", TimeNotSynchronized), (\"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\", TimeNotSynchronized), (\"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\", TimeNotSynchronized), (\"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d\", TimeNotSynchronized), (\"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\", TimeNotSynchronized), (\"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\", TimeNotSynchronized), (\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\", TimeNotSynchronized), (\"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139\", TimeNotSynchronized), (\"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\", TimeNotSynchronized)]","latency_us":104163,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.094452023Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"172.397744ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3a720b85-9fad-498b-8b55-af5171475da6\", \"content-length\": \"124\", \"date\": \"Mon, 18 Nov 2024 20:59:24 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3a720b85-9fad-498b-8b55-af5171475da6\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 62e83095-0557-4ac0-8b12-7429e79e4bb6 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4554ce4c-90f4-4fd2-93bd-086596238122 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: e9e770f2-88d4-4893-909a-79400af2db8b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b99c98c4-f6a4-43cc-a789-4b40c33f469f, kind: Service { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 }, name: Name(\"ntp-5066b4aa-939d-40f6-9e2e-cc4b97bad1b4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 130, 96])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ddac3753-9787-478c-a9ea-d655291375e0 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3eae245b-f0e3-4234-b3b9-d9b4de089123 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5d056c46-671e-47fd-ab76-2549bb8d565b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: e786554c-6b77-4063-92f2-2aeb18d93f3d (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 14e637a2-51ce-4cac-b831-85b757e5c55d (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 95dce781-7d6b-4bf1-88f1-880b60d0adba, kind: Service { id: 14e637a2-51ce-4cac-b831-85b757e5c55d }, name: Name(\"external-dns-14e637a2-51ce-4cac-b831-85b757e5c55d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 238])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3b3dea8a-e84f-4acf-bafc-3b0e4a815dc5, kind: Service { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 }, name: Name(\"external-dns-be35e232-dd9b-4d59-b4e1-eadb7f69bb77\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 109])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: db0c0ea5-0d8f-4f9d-9979-f3989ce962ea, kind: Service { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 }, name: Name(\"nexus-16a7178a-dbb6-4392-b395-7bddb3817d13\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9b2d2174-b236-4546-bace-300e05afe3c2 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 5dbd0a76-35d9-4b29-8800-824968d1caba, kind: Service { id: 9b2d2174-b236-4546-bace-300e05afe3c2 }, name: Name(\"nexus-9b2d2174-b236-4546-bace-300e05afe3c2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 121])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e67bea26-1a67-4166-875f-3cab431644e6, kind: Service { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 }, name: Name(\"nexus-3dce42ed-74cb-4c8b-9326-8273a1efd563\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 239])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 84b5a094-9eb0-4ff8-be58-5c4913059740 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e5181def-6535-4350-b7f4-acf3785cf121 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b77d62e-b462-4293-950d-00a565b722bd (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 40c1b925-76fc-473c-9317-ffcfbf8aef2d (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 154903a9-f4ab-4352-b947-96c5fce154b3 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b5ceaf3-96d0-4f36-882b-965f9acc31c4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: c7378503-c5b8-4eee-91c0-2c15e3708aea (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 090b4de4-d9d1-47aa-96a9-4cc472900139 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: abc9c059-37e5-408c-b8e0-9fdb2f94fcfb (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ced4eec8-4bf4-46b3-843c-91242defe60a (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 08aaade2-f75a-444c-b97a-b4d3bf97b095 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External } } } }, OmicronZoneConfig { id: 04a81970-7129-40c7-b6f5-36219a8ad854 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cdeb736-9e69-46f9-afb0-59255807dbe4 (service), filesystem_pool: Some(ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External } } } }, OmicronZoneConfig { id: 948190c1-2519-4134-a120-8d6900d50730 (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:25.268079301Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:25.268523315Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:25.268559545Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:25.268711056Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:25.268819357Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:25.268829177Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:25.268983438Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:25.269097199Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:25.269146829Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.364933836Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.364991616Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.364998276Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.365004516Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.365008636Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.365014356Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.365021176Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.365026216Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.365031616Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.365054167Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.365059677Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.365065027Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.365070397Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.365077547Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.365083367Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.365089887Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.365095327Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.365102387Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.365105967Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:25.365186117Z","hostname":"sock","pid":1574,"uri":"/omicron-zones","method":"PUT","req_id":"c24442be-e5c2-4be8-b6fc-4535d1251da1","remote_addr":"[fd00:1122:3344:101::1]:49408","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139\", TimeNotSynchronized), (\"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\", TimeNotSynchronized), (\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\", TimeNotSynchronized), (\"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2\", TimeNotSynchronized), (\"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\", TimeNotSynchronized), (\"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\", TimeNotSynchronized), (\"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\", TimeNotSynchronized), (\"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\", TimeNotSynchronized), (\"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3\", TimeNotSynchronized), (\"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\", TimeNotSynchronized), (\"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121\", TimeNotSynchronized), (\"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d\", TimeNotSynchronized), (\"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\", TimeNotSynchronized), (\"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\", TimeNotSynchronized), (\"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\", TimeNotSynchronized), (\"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\", TimeNotSynchronized), (\"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\", TimeNotSynchronized), (\"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\", TimeNotSynchronized), (\"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\", TimeNotSynchronized)]","latency_us":97263,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.365360989Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"220.532384ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c24442be-e5c2-4be8-b6fc-4535d1251da1\", \"content-length\": \"124\", \"date\": \"Mon, 18 Nov 2024 20:59:24 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c24442be-e5c2-4be8-b6fc-4535d1251da1\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 62e83095-0557-4ac0-8b12-7429e79e4bb6 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4554ce4c-90f4-4fd2-93bd-086596238122 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: e9e770f2-88d4-4893-909a-79400af2db8b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b99c98c4-f6a4-43cc-a789-4b40c33f469f, kind: Service { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 }, name: Name(\"ntp-5066b4aa-939d-40f6-9e2e-cc4b97bad1b4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 130, 96])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ddac3753-9787-478c-a9ea-d655291375e0 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3eae245b-f0e3-4234-b3b9-d9b4de089123 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5d056c46-671e-47fd-ab76-2549bb8d565b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: e786554c-6b77-4063-92f2-2aeb18d93f3d (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 14e637a2-51ce-4cac-b831-85b757e5c55d (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 95dce781-7d6b-4bf1-88f1-880b60d0adba, kind: Service { id: 14e637a2-51ce-4cac-b831-85b757e5c55d }, name: Name(\"external-dns-14e637a2-51ce-4cac-b831-85b757e5c55d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 238])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3b3dea8a-e84f-4acf-bafc-3b0e4a815dc5, kind: Service { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 }, name: Name(\"external-dns-be35e232-dd9b-4d59-b4e1-eadb7f69bb77\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 109])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: db0c0ea5-0d8f-4f9d-9979-f3989ce962ea, kind: Service { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 }, name: Name(\"nexus-16a7178a-dbb6-4392-b395-7bddb3817d13\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9b2d2174-b236-4546-bace-300e05afe3c2 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 5dbd0a76-35d9-4b29-8800-824968d1caba, kind: Service { id: 9b2d2174-b236-4546-bace-300e05afe3c2 }, name: Name(\"nexus-9b2d2174-b236-4546-bace-300e05afe3c2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 121])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e67bea26-1a67-4166-875f-3cab431644e6, kind: Service { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 }, name: Name(\"nexus-3dce42ed-74cb-4c8b-9326-8273a1efd563\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 239])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 84b5a094-9eb0-4ff8-be58-5c4913059740 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e5181def-6535-4350-b7f4-acf3785cf121 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b77d62e-b462-4293-950d-00a565b722bd (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 40c1b925-76fc-473c-9317-ffcfbf8aef2d (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 154903a9-f4ab-4352-b947-96c5fce154b3 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b5ceaf3-96d0-4f36-882b-965f9acc31c4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: c7378503-c5b8-4eee-91c0-2c15e3708aea (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 090b4de4-d9d1-47aa-96a9-4cc472900139 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: abc9c059-37e5-408c-b8e0-9fdb2f94fcfb (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ced4eec8-4bf4-46b3-843c-91242defe60a (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 08aaade2-f75a-444c-b97a-b4d3bf97b095 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External } } } }, OmicronZoneConfig { id: 04a81970-7129-40c7-b6f5-36219a8ad854 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cdeb736-9e69-46f9-afb0-59255807dbe4 (service), filesystem_pool: Some(ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External } } } }, OmicronZoneConfig { id: 948190c1-2519-4134-a120-8d6900d50730 (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:25.58620461Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:25.586496783Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:25.586541263Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:25.586651444Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:25.586788995Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:25.586805765Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:25.586936666Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:25.587054247Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:25.587111637Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.69042525Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.69048226Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.69050147Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.690506241Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.690510481Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.690517741Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.690523431Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.690528921Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.690534461Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.690541401Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.690546831Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.690552221Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.690558951Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.690564481Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.690568491Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.690575201Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.690580151Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.690599011Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.690604441Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:25.690701682Z","hostname":"sock","pid":1574,"uri":"/omicron-zones","method":"PUT","req_id":"e6bc8df8-69d0-4741-bf60-6c91a52e05f4","remote_addr":"[fd00:1122:3344:101::1]:49408","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\", TimeNotSynchronized), (\"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139\", TimeNotSynchronized), (\"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\", TimeNotSynchronized), (\"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\", TimeNotSynchronized), (\"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121\", TimeNotSynchronized), (\"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\", TimeNotSynchronized), (\"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\", TimeNotSynchronized), (\"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2\", TimeNotSynchronized), (\"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3\", TimeNotSynchronized), (\"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\", TimeNotSynchronized), (\"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\", TimeNotSynchronized), (\"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d\", TimeNotSynchronized), (\"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\", TimeNotSynchronized), (\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\", TimeNotSynchronized), (\"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\", TimeNotSynchronized), (\"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\", TimeNotSynchronized), (\"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\", TimeNotSynchronized), (\"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\", TimeNotSynchronized), (\"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\", TimeNotSynchronized)]","latency_us":104650,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:25.690859263Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"245.576753ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e6bc8df8-69d0-4741-bf60-6c91a52e05f4\", \"content-length\": \"124\", \"date\": \"Mon, 18 Nov 2024 20:59:24 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e6bc8df8-69d0-4741-bf60-6c91a52e05f4\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 62e83095-0557-4ac0-8b12-7429e79e4bb6 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4554ce4c-90f4-4fd2-93bd-086596238122 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: e9e770f2-88d4-4893-909a-79400af2db8b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b99c98c4-f6a4-43cc-a789-4b40c33f469f, kind: Service { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 }, name: Name(\"ntp-5066b4aa-939d-40f6-9e2e-cc4b97bad1b4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 130, 96])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ddac3753-9787-478c-a9ea-d655291375e0 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3eae245b-f0e3-4234-b3b9-d9b4de089123 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5d056c46-671e-47fd-ab76-2549bb8d565b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: e786554c-6b77-4063-92f2-2aeb18d93f3d (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 14e637a2-51ce-4cac-b831-85b757e5c55d (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 95dce781-7d6b-4bf1-88f1-880b60d0adba, kind: Service { id: 14e637a2-51ce-4cac-b831-85b757e5c55d }, name: Name(\"external-dns-14e637a2-51ce-4cac-b831-85b757e5c55d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 238])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3b3dea8a-e84f-4acf-bafc-3b0e4a815dc5, kind: Service { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 }, name: Name(\"external-dns-be35e232-dd9b-4d59-b4e1-eadb7f69bb77\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 109])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: db0c0ea5-0d8f-4f9d-9979-f3989ce962ea, kind: Service { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 }, name: Name(\"nexus-16a7178a-dbb6-4392-b395-7bddb3817d13\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9b2d2174-b236-4546-bace-300e05afe3c2 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 5dbd0a76-35d9-4b29-8800-824968d1caba, kind: Service { id: 9b2d2174-b236-4546-bace-300e05afe3c2 }, name: Name(\"nexus-9b2d2174-b236-4546-bace-300e05afe3c2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 121])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e67bea26-1a67-4166-875f-3cab431644e6, kind: Service { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 }, name: Name(\"nexus-3dce42ed-74cb-4c8b-9326-8273a1efd563\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 239])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 84b5a094-9eb0-4ff8-be58-5c4913059740 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e5181def-6535-4350-b7f4-acf3785cf121 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b77d62e-b462-4293-950d-00a565b722bd (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 40c1b925-76fc-473c-9317-ffcfbf8aef2d (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 154903a9-f4ab-4352-b947-96c5fce154b3 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b5ceaf3-96d0-4f36-882b-965f9acc31c4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: c7378503-c5b8-4eee-91c0-2c15e3708aea (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 090b4de4-d9d1-47aa-96a9-4cc472900139 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: abc9c059-37e5-408c-b8e0-9fdb2f94fcfb (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ced4eec8-4bf4-46b3-843c-91242defe60a (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 08aaade2-f75a-444c-b97a-b4d3bf97b095 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External } } } }, OmicronZoneConfig { id: 04a81970-7129-40c7-b6f5-36219a8ad854 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cdeb736-9e69-46f9-afb0-59255807dbe4 (service), filesystem_pool: Some(ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External } } } }, OmicronZoneConfig { id: 948190c1-2519-4134-a120-8d6900d50730 (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:25.937700749Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:25.937983511Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:25.938029831Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:25.938175012Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:25.938296553Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:25.938313413Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:25.938500745Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:25.938645486Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:25.938699696Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.045188653Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.045275633Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.045289263Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.045300074Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.045308923Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.045319654Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.045325604Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.045331164Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.045338174Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.045351094Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.045356594Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.045360294Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.045364084Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.045369314Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.045374664Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.045380104Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.045385164Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.045390504Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.045395744Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:26.045449385Z","hostname":"sock","pid":1574,"uri":"/omicron-zones","method":"PUT","req_id":"50a85330-84b1-4eb5-81e8-05936e16892d","remote_addr":"[fd00:1122:3344:101::1]:49408","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3\", TimeNotSynchronized), (\"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\", TimeNotSynchronized), (\"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139\", TimeNotSynchronized), (\"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\", TimeNotSynchronized), (\"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\", TimeNotSynchronized), (\"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\", TimeNotSynchronized), (\"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\", TimeNotSynchronized), (\"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\", TimeNotSynchronized), (\"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\", TimeNotSynchronized), (\"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121\", TimeNotSynchronized), (\"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\", TimeNotSynchronized), (\"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\", TimeNotSynchronized), (\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\", TimeNotSynchronized), (\"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\", TimeNotSynchronized), (\"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\", TimeNotSynchronized), (\"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d\", TimeNotSynchronized), (\"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\", TimeNotSynchronized), (\"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2\", TimeNotSynchronized), (\"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\", TimeNotSynchronized)]","latency_us":107927,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.045620066Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"151.955496ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"50a85330-84b1-4eb5-81e8-05936e16892d\", \"content-length\": \"124\", \"date\": \"Mon, 18 Nov 2024 20:59:25 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"50a85330-84b1-4eb5-81e8-05936e16892d\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 62e83095-0557-4ac0-8b12-7429e79e4bb6 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4554ce4c-90f4-4fd2-93bd-086596238122 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: e9e770f2-88d4-4893-909a-79400af2db8b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b99c98c4-f6a4-43cc-a789-4b40c33f469f, kind: Service { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 }, name: Name(\"ntp-5066b4aa-939d-40f6-9e2e-cc4b97bad1b4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 130, 96])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ddac3753-9787-478c-a9ea-d655291375e0 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3eae245b-f0e3-4234-b3b9-d9b4de089123 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5d056c46-671e-47fd-ab76-2549bb8d565b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: e786554c-6b77-4063-92f2-2aeb18d93f3d (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 14e637a2-51ce-4cac-b831-85b757e5c55d (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 95dce781-7d6b-4bf1-88f1-880b60d0adba, kind: Service { id: 14e637a2-51ce-4cac-b831-85b757e5c55d }, name: Name(\"external-dns-14e637a2-51ce-4cac-b831-85b757e5c55d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 238])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3b3dea8a-e84f-4acf-bafc-3b0e4a815dc5, kind: Service { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 }, name: Name(\"external-dns-be35e232-dd9b-4d59-b4e1-eadb7f69bb77\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 109])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: db0c0ea5-0d8f-4f9d-9979-f3989ce962ea, kind: Service { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 }, name: Name(\"nexus-16a7178a-dbb6-4392-b395-7bddb3817d13\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9b2d2174-b236-4546-bace-300e05afe3c2 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 5dbd0a76-35d9-4b29-8800-824968d1caba, kind: Service { id: 9b2d2174-b236-4546-bace-300e05afe3c2 }, name: Name(\"nexus-9b2d2174-b236-4546-bace-300e05afe3c2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 121])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e67bea26-1a67-4166-875f-3cab431644e6, kind: Service { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 }, name: Name(\"nexus-3dce42ed-74cb-4c8b-9326-8273a1efd563\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 239])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 84b5a094-9eb0-4ff8-be58-5c4913059740 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e5181def-6535-4350-b7f4-acf3785cf121 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b77d62e-b462-4293-950d-00a565b722bd (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 40c1b925-76fc-473c-9317-ffcfbf8aef2d (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 154903a9-f4ab-4352-b947-96c5fce154b3 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b5ceaf3-96d0-4f36-882b-965f9acc31c4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: c7378503-c5b8-4eee-91c0-2c15e3708aea (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 090b4de4-d9d1-47aa-96a9-4cc472900139 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: abc9c059-37e5-408c-b8e0-9fdb2f94fcfb (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ced4eec8-4bf4-46b3-843c-91242defe60a (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 08aaade2-f75a-444c-b97a-b4d3bf97b095 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External } } } }, OmicronZoneConfig { id: 04a81970-7129-40c7-b6f5-36219a8ad854 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cdeb736-9e69-46f9-afb0-59255807dbe4 (service), filesystem_pool: Some(ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External } } } }, OmicronZoneConfig { id: 948190c1-2519-4134-a120-8d6900d50730 (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:26.198491269Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:26.198800211Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:26.198834682Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:26.198958633Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:26.199206054Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:26.199222355Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:26.199368826Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:26.199496557Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:26.199574627Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.300037719Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.300100329Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.300124389Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.300130639Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.300136129Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.300143289Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.300147289Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.30015422Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.300160389Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.300165529Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.30017244Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.30017802Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.3001847Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.30019Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.30019586Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.30020129Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.30020648Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.30021609Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.30022157Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:26.30029601Z","hostname":"sock","pid":1574,"uri":"/omicron-zones","method":"PUT","req_id":"50f3e323-3f1e-4c99-841e-6ea49d7deb83","remote_addr":"[fd00:1122:3344:101::1]:49408","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\", TimeNotSynchronized), (\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\", TimeNotSynchronized), (\"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2\", TimeNotSynchronized), (\"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\", TimeNotSynchronized), (\"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\", TimeNotSynchronized), (\"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\", TimeNotSynchronized), (\"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\", TimeNotSynchronized), (\"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\", TimeNotSynchronized), (\"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\", TimeNotSynchronized), (\"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121\", TimeNotSynchronized), (\"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\", TimeNotSynchronized), (\"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139\", TimeNotSynchronized), (\"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\", TimeNotSynchronized), (\"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d\", TimeNotSynchronized), (\"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3\", TimeNotSynchronized), (\"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\", TimeNotSynchronized), (\"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\", TimeNotSynchronized), (\"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\", TimeNotSynchronized), (\"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\", TimeNotSynchronized)]","latency_us":101957,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.300462682Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"282.784454ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"50f3e323-3f1e-4c99-841e-6ea49d7deb83\", \"content-length\": \"124\", \"date\": \"Mon, 18 Nov 2024 20:59:25 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"50f3e323-3f1e-4c99-841e-6ea49d7deb83\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 62e83095-0557-4ac0-8b12-7429e79e4bb6 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4554ce4c-90f4-4fd2-93bd-086596238122 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: e9e770f2-88d4-4893-909a-79400af2db8b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b99c98c4-f6a4-43cc-a789-4b40c33f469f, kind: Service { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 }, name: Name(\"ntp-5066b4aa-939d-40f6-9e2e-cc4b97bad1b4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 130, 96])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ddac3753-9787-478c-a9ea-d655291375e0 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3eae245b-f0e3-4234-b3b9-d9b4de089123 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5d056c46-671e-47fd-ab76-2549bb8d565b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: e786554c-6b77-4063-92f2-2aeb18d93f3d (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 14e637a2-51ce-4cac-b831-85b757e5c55d (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 95dce781-7d6b-4bf1-88f1-880b60d0adba, kind: Service { id: 14e637a2-51ce-4cac-b831-85b757e5c55d }, name: Name(\"external-dns-14e637a2-51ce-4cac-b831-85b757e5c55d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 238])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3b3dea8a-e84f-4acf-bafc-3b0e4a815dc5, kind: Service { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 }, name: Name(\"external-dns-be35e232-dd9b-4d59-b4e1-eadb7f69bb77\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 109])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: db0c0ea5-0d8f-4f9d-9979-f3989ce962ea, kind: Service { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 }, name: Name(\"nexus-16a7178a-dbb6-4392-b395-7bddb3817d13\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9b2d2174-b236-4546-bace-300e05afe3c2 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 5dbd0a76-35d9-4b29-8800-824968d1caba, kind: Service { id: 9b2d2174-b236-4546-bace-300e05afe3c2 }, name: Name(\"nexus-9b2d2174-b236-4546-bace-300e05afe3c2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 121])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e67bea26-1a67-4166-875f-3cab431644e6, kind: Service { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 }, name: Name(\"nexus-3dce42ed-74cb-4c8b-9326-8273a1efd563\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 239])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 84b5a094-9eb0-4ff8-be58-5c4913059740 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e5181def-6535-4350-b7f4-acf3785cf121 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b77d62e-b462-4293-950d-00a565b722bd (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 40c1b925-76fc-473c-9317-ffcfbf8aef2d (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 154903a9-f4ab-4352-b947-96c5fce154b3 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b5ceaf3-96d0-4f36-882b-965f9acc31c4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: c7378503-c5b8-4eee-91c0-2c15e3708aea (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 090b4de4-d9d1-47aa-96a9-4cc472900139 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: abc9c059-37e5-408c-b8e0-9fdb2f94fcfb (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ced4eec8-4bf4-46b3-843c-91242defe60a (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 08aaade2-f75a-444c-b97a-b4d3bf97b095 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External } } } }, OmicronZoneConfig { id: 04a81970-7129-40c7-b6f5-36219a8ad854 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cdeb736-9e69-46f9-afb0-59255807dbe4 (service), filesystem_pool: Some(ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External } } } }, OmicronZoneConfig { id: 948190c1-2519-4134-a120-8d6900d50730 (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:26.583028175Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:26.583303107Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:26.583342547Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:26.583473598Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:26.583588219Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:26.583603219Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:26.58375071Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:26.583871961Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:26.583927292Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.683567527Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.683612117Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.683617027Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.683621237Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.683625427Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.683629707Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.683633677Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.683637657Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.683641437Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.683645297Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.683661957Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.683665648Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.683669577Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.683673237Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.683677018Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.683680808Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.683684378Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.683729738Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.683736438Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:26.683868079Z","hostname":"sock","pid":1574,"uri":"/omicron-zones","method":"PUT","req_id":"f5e36c84-31a2-4453-8f85-8dff6b924ff3","remote_addr":"[fd00:1122:3344:101::1]:49408","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3\", TimeNotSynchronized), (\"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\", TimeNotSynchronized), (\"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\", TimeNotSynchronized), (\"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\", TimeNotSynchronized), (\"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139\", TimeNotSynchronized), (\"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\", TimeNotSynchronized), (\"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d\", TimeNotSynchronized), (\"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\", TimeNotSynchronized), (\"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2\", TimeNotSynchronized), (\"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121\", TimeNotSynchronized), (\"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\", TimeNotSynchronized), (\"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\", TimeNotSynchronized), (\"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\", TimeNotSynchronized), (\"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\", TimeNotSynchronized), (\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\", TimeNotSynchronized), (\"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\", TimeNotSynchronized), (\"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\", TimeNotSynchronized), (\"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\", TimeNotSynchronized), (\"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\", TimeNotSynchronized)]","latency_us":101010,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:26.68403645Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"293.692713ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f5e36c84-31a2-4453-8f85-8dff6b924ff3\", \"content-length\": \"124\", \"date\": \"Mon, 18 Nov 2024 20:59:26 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f5e36c84-31a2-4453-8f85-8dff6b924ff3\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 62e83095-0557-4ac0-8b12-7429e79e4bb6 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4554ce4c-90f4-4fd2-93bd-086596238122 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: e9e770f2-88d4-4893-909a-79400af2db8b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b99c98c4-f6a4-43cc-a789-4b40c33f469f, kind: Service { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 }, name: Name(\"ntp-5066b4aa-939d-40f6-9e2e-cc4b97bad1b4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 130, 96])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ddac3753-9787-478c-a9ea-d655291375e0 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3eae245b-f0e3-4234-b3b9-d9b4de089123 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5d056c46-671e-47fd-ab76-2549bb8d565b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: e786554c-6b77-4063-92f2-2aeb18d93f3d (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 14e637a2-51ce-4cac-b831-85b757e5c55d (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 95dce781-7d6b-4bf1-88f1-880b60d0adba, kind: Service { id: 14e637a2-51ce-4cac-b831-85b757e5c55d }, name: Name(\"external-dns-14e637a2-51ce-4cac-b831-85b757e5c55d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 238])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3b3dea8a-e84f-4acf-bafc-3b0e4a815dc5, kind: Service { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 }, name: Name(\"external-dns-be35e232-dd9b-4d59-b4e1-eadb7f69bb77\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 109])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: db0c0ea5-0d8f-4f9d-9979-f3989ce962ea, kind: Service { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 }, name: Name(\"nexus-16a7178a-dbb6-4392-b395-7bddb3817d13\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9b2d2174-b236-4546-bace-300e05afe3c2 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 5dbd0a76-35d9-4b29-8800-824968d1caba, kind: Service { id: 9b2d2174-b236-4546-bace-300e05afe3c2 }, name: Name(\"nexus-9b2d2174-b236-4546-bace-300e05afe3c2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 121])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e67bea26-1a67-4166-875f-3cab431644e6, kind: Service { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 }, name: Name(\"nexus-3dce42ed-74cb-4c8b-9326-8273a1efd563\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 239])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 84b5a094-9eb0-4ff8-be58-5c4913059740 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e5181def-6535-4350-b7f4-acf3785cf121 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b77d62e-b462-4293-950d-00a565b722bd (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 40c1b925-76fc-473c-9317-ffcfbf8aef2d (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 154903a9-f4ab-4352-b947-96c5fce154b3 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b5ceaf3-96d0-4f36-882b-965f9acc31c4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: c7378503-c5b8-4eee-91c0-2c15e3708aea (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 090b4de4-d9d1-47aa-96a9-4cc472900139 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: abc9c059-37e5-408c-b8e0-9fdb2f94fcfb (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ced4eec8-4bf4-46b3-843c-91242defe60a (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 08aaade2-f75a-444c-b97a-b4d3bf97b095 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External } } } }, OmicronZoneConfig { id: 04a81970-7129-40c7-b6f5-36219a8ad854 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cdeb736-9e69-46f9-afb0-59255807dbe4 (service), filesystem_pool: Some(ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External } } } }, OmicronZoneConfig { id: 948190c1-2519-4134-a120-8d6900d50730 (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:26.978535833Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:26.978814155Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:26.978852005Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:26.979094797Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:26.979246628Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:26.979264478Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:26.979412789Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:26.97955781Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:26.979626671Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:27.085350121Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:27.085415262Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:27.085432912Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:27.085439202Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:27.085446422Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:27.085450602Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:27.085457402Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:27.085462382Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:27.085467772Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:27.085473202Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:27.085479732Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:27.085485262Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:27.085490662Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:27.085496153Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:27.085500553Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:27.085505762Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:27.085509353Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:27.085526263Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:27.085531393Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:27.085588673Z","hostname":"sock","pid":1574,"uri":"/omicron-zones","method":"PUT","req_id":"059d388a-4ddf-421b-9b08-03848194d9f4","remote_addr":"[fd00:1122:3344:101::1]:49408","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\", TimeNotSynchronized), (\"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\", TimeNotSynchronized), (\"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121\", TimeNotSynchronized), (\"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\", TimeNotSynchronized), (\"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139\", TimeNotSynchronized), (\"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\", TimeNotSynchronized), (\"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3\", TimeNotSynchronized), (\"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2\", TimeNotSynchronized), (\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\", TimeNotSynchronized), (\"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d\", TimeNotSynchronized), (\"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\", TimeNotSynchronized), (\"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\", TimeNotSynchronized), (\"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\", TimeNotSynchronized), (\"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\", TimeNotSynchronized), (\"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\", TimeNotSynchronized), (\"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\", TimeNotSynchronized), (\"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\", TimeNotSynchronized), (\"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\", TimeNotSynchronized), (\"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\", TimeNotSynchronized)]","latency_us":107237,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:27.085760055Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"470.524704ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"059d388a-4ddf-421b-9b08-03848194d9f4\", \"content-length\": \"124\", \"date\": \"Mon, 18 Nov 2024 20:59:27 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"059d388a-4ddf-421b-9b08-03848194d9f4\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 62e83095-0557-4ac0-8b12-7429e79e4bb6 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4554ce4c-90f4-4fd2-93bd-086596238122 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: e9e770f2-88d4-4893-909a-79400af2db8b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b99c98c4-f6a4-43cc-a789-4b40c33f469f, kind: Service { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 }, name: Name(\"ntp-5066b4aa-939d-40f6-9e2e-cc4b97bad1b4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 130, 96])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ddac3753-9787-478c-a9ea-d655291375e0 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3eae245b-f0e3-4234-b3b9-d9b4de089123 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5d056c46-671e-47fd-ab76-2549bb8d565b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: e786554c-6b77-4063-92f2-2aeb18d93f3d (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 14e637a2-51ce-4cac-b831-85b757e5c55d (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 95dce781-7d6b-4bf1-88f1-880b60d0adba, kind: Service { id: 14e637a2-51ce-4cac-b831-85b757e5c55d }, name: Name(\"external-dns-14e637a2-51ce-4cac-b831-85b757e5c55d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 238])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3b3dea8a-e84f-4acf-bafc-3b0e4a815dc5, kind: Service { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 }, name: Name(\"external-dns-be35e232-dd9b-4d59-b4e1-eadb7f69bb77\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 109])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: db0c0ea5-0d8f-4f9d-9979-f3989ce962ea, kind: Service { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 }, name: Name(\"nexus-16a7178a-dbb6-4392-b395-7bddb3817d13\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9b2d2174-b236-4546-bace-300e05afe3c2 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 5dbd0a76-35d9-4b29-8800-824968d1caba, kind: Service { id: 9b2d2174-b236-4546-bace-300e05afe3c2 }, name: Name(\"nexus-9b2d2174-b236-4546-bace-300e05afe3c2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 121])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e67bea26-1a67-4166-875f-3cab431644e6, kind: Service { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 }, name: Name(\"nexus-3dce42ed-74cb-4c8b-9326-8273a1efd563\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 239])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 84b5a094-9eb0-4ff8-be58-5c4913059740 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e5181def-6535-4350-b7f4-acf3785cf121 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b77d62e-b462-4293-950d-00a565b722bd (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 40c1b925-76fc-473c-9317-ffcfbf8aef2d (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 154903a9-f4ab-4352-b947-96c5fce154b3 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b5ceaf3-96d0-4f36-882b-965f9acc31c4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: c7378503-c5b8-4eee-91c0-2c15e3708aea (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 090b4de4-d9d1-47aa-96a9-4cc472900139 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: abc9c059-37e5-408c-b8e0-9fdb2f94fcfb (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ced4eec8-4bf4-46b3-843c-91242defe60a (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 08aaade2-f75a-444c-b97a-b4d3bf97b095 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External } } } }, OmicronZoneConfig { id: 04a81970-7129-40c7-b6f5-36219a8ad854 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cdeb736-9e69-46f9-afb0-59255807dbe4 (service), filesystem_pool: Some(ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External } } } }, OmicronZoneConfig { id: 948190c1-2519-4134-a120-8d6900d50730 (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:27.556078432Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:27.556352964Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:27.556396604Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:27.556527385Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:27.556647076Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:27.556659626Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:27.556802577Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:27.556912188Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:27.556970598Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:27.650136075Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:27.650193816Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:27.650202056Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:27.650207566Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:27.650213086Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:27.650220266Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:27.650226456Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:27.650232216Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:27.650237556Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:27.650243646Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:27.650256606Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:27.650263446Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:27.650268626Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:27.650273816Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:27.650279006Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:27.650286046Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:27.650291446Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:27.650296866Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:27.650302436Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:27.650416467Z","hostname":"sock","pid":1574,"uri":"/omicron-zones","method":"PUT","req_id":"73f30ecb-5244-4cd9-8f14-c6e566d4463d","remote_addr":"[fd00:1122:3344:101::1]:49408","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\", TimeNotSynchronized), (\"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\", TimeNotSynchronized), (\"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\", TimeNotSynchronized), (\"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\", TimeNotSynchronized), (\"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121\", TimeNotSynchronized), (\"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3\", TimeNotSynchronized), (\"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\", TimeNotSynchronized), (\"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\", TimeNotSynchronized), (\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\", TimeNotSynchronized), (\"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2\", TimeNotSynchronized), (\"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\", TimeNotSynchronized), (\"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139\", TimeNotSynchronized), (\"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\", TimeNotSynchronized), (\"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\", TimeNotSynchronized), (\"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d\", TimeNotSynchronized), (\"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\", TimeNotSynchronized), (\"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\", TimeNotSynchronized), (\"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\", TimeNotSynchronized), (\"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\", TimeNotSynchronized)]","latency_us":94481,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:27.650586088Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"279.590586ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"73f30ecb-5244-4cd9-8f14-c6e566d4463d\", \"content-length\": \"124\", \"date\": \"Mon, 18 Nov 2024 20:59:26 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"73f30ecb-5244-4cd9-8f14-c6e566d4463d\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 62e83095-0557-4ac0-8b12-7429e79e4bb6 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4554ce4c-90f4-4fd2-93bd-086596238122 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: e9e770f2-88d4-4893-909a-79400af2db8b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b99c98c4-f6a4-43cc-a789-4b40c33f469f, kind: Service { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 }, name: Name(\"ntp-5066b4aa-939d-40f6-9e2e-cc4b97bad1b4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 130, 96])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ddac3753-9787-478c-a9ea-d655291375e0 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3eae245b-f0e3-4234-b3b9-d9b4de089123 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5d056c46-671e-47fd-ab76-2549bb8d565b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: e786554c-6b77-4063-92f2-2aeb18d93f3d (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 14e637a2-51ce-4cac-b831-85b757e5c55d (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 95dce781-7d6b-4bf1-88f1-880b60d0adba, kind: Service { id: 14e637a2-51ce-4cac-b831-85b757e5c55d }, name: Name(\"external-dns-14e637a2-51ce-4cac-b831-85b757e5c55d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 238])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3b3dea8a-e84f-4acf-bafc-3b0e4a815dc5, kind: Service { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 }, name: Name(\"external-dns-be35e232-dd9b-4d59-b4e1-eadb7f69bb77\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 109])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: db0c0ea5-0d8f-4f9d-9979-f3989ce962ea, kind: Service { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 }, name: Name(\"nexus-16a7178a-dbb6-4392-b395-7bddb3817d13\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9b2d2174-b236-4546-bace-300e05afe3c2 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 5dbd0a76-35d9-4b29-8800-824968d1caba, kind: Service { id: 9b2d2174-b236-4546-bace-300e05afe3c2 }, name: Name(\"nexus-9b2d2174-b236-4546-bace-300e05afe3c2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 121])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e67bea26-1a67-4166-875f-3cab431644e6, kind: Service { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 }, name: Name(\"nexus-3dce42ed-74cb-4c8b-9326-8273a1efd563\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 239])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 84b5a094-9eb0-4ff8-be58-5c4913059740 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e5181def-6535-4350-b7f4-acf3785cf121 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b77d62e-b462-4293-950d-00a565b722bd (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 40c1b925-76fc-473c-9317-ffcfbf8aef2d (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 154903a9-f4ab-4352-b947-96c5fce154b3 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b5ceaf3-96d0-4f36-882b-965f9acc31c4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: c7378503-c5b8-4eee-91c0-2c15e3708aea (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 090b4de4-d9d1-47aa-96a9-4cc472900139 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: abc9c059-37e5-408c-b8e0-9fdb2f94fcfb (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ced4eec8-4bf4-46b3-843c-91242defe60a (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 08aaade2-f75a-444c-b97a-b4d3bf97b095 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External } } } }, OmicronZoneConfig { id: 04a81970-7129-40c7-b6f5-36219a8ad854 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cdeb736-9e69-46f9-afb0-59255807dbe4 (service), filesystem_pool: Some(ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External } } } }, OmicronZoneConfig { id: 948190c1-2519-4134-a120-8d6900d50730 (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:27.931244907Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:27.931513429Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:27.931564039Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:27.93169976Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:27.931835661Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:27.931854442Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:27.932004171Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:27.932140453Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:27.932199664Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:28.025906675Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:28.025946465Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:28.025967205Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:28.025974845Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:28.025980905Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:28.025986706Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:28.025993886Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:28.025999895Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:28.026007046Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:28.026013046Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:28.026018646Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:28.026024296Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:28.026029826Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:28.026036876Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:28.026042656Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:28.026048366Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:28.026053646Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:28.026064426Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:28.026068406Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:28.026147677Z","hostname":"sock","pid":1574,"uri":"/omicron-zones","method":"PUT","req_id":"ffdcd050-960e-434f-880d-654060f72181","remote_addr":"[fd00:1122:3344:101::1]:49408","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121\", TimeNotSynchronized), (\"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\", TimeNotSynchronized), (\"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\", TimeNotSynchronized), (\"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2\", TimeNotSynchronized), (\"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3\", TimeNotSynchronized), (\"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\", TimeNotSynchronized), (\"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\", TimeNotSynchronized), (\"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\", TimeNotSynchronized), (\"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\", TimeNotSynchronized), (\"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\", TimeNotSynchronized), (\"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\", TimeNotSynchronized), (\"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\", TimeNotSynchronized), (\"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\", TimeNotSynchronized), (\"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\", TimeNotSynchronized), (\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\", TimeNotSynchronized), (\"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\", TimeNotSynchronized), (\"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\", TimeNotSynchronized), (\"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139\", TimeNotSynchronized), (\"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d\", TimeNotSynchronized)]","latency_us":95056,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:28.026323048Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"499.344428ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ffdcd050-960e-434f-880d-654060f72181\", \"content-length\": \"124\", \"date\": \"Mon, 18 Nov 2024 20:59:27 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ffdcd050-960e-434f-880d-654060f72181\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 62e83095-0557-4ac0-8b12-7429e79e4bb6 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4554ce4c-90f4-4fd2-93bd-086596238122 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: e9e770f2-88d4-4893-909a-79400af2db8b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b99c98c4-f6a4-43cc-a789-4b40c33f469f, kind: Service { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 }, name: Name(\"ntp-5066b4aa-939d-40f6-9e2e-cc4b97bad1b4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 130, 96])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ddac3753-9787-478c-a9ea-d655291375e0 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3eae245b-f0e3-4234-b3b9-d9b4de089123 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5d056c46-671e-47fd-ab76-2549bb8d565b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: e786554c-6b77-4063-92f2-2aeb18d93f3d (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 14e637a2-51ce-4cac-b831-85b757e5c55d (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 95dce781-7d6b-4bf1-88f1-880b60d0adba, kind: Service { id: 14e637a2-51ce-4cac-b831-85b757e5c55d }, name: Name(\"external-dns-14e637a2-51ce-4cac-b831-85b757e5c55d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 238])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3b3dea8a-e84f-4acf-bafc-3b0e4a815dc5, kind: Service { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 }, name: Name(\"external-dns-be35e232-dd9b-4d59-b4e1-eadb7f69bb77\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 109])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: db0c0ea5-0d8f-4f9d-9979-f3989ce962ea, kind: Service { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 }, name: Name(\"nexus-16a7178a-dbb6-4392-b395-7bddb3817d13\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9b2d2174-b236-4546-bace-300e05afe3c2 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 5dbd0a76-35d9-4b29-8800-824968d1caba, kind: Service { id: 9b2d2174-b236-4546-bace-300e05afe3c2 }, name: Name(\"nexus-9b2d2174-b236-4546-bace-300e05afe3c2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 121])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e67bea26-1a67-4166-875f-3cab431644e6, kind: Service { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 }, name: Name(\"nexus-3dce42ed-74cb-4c8b-9326-8273a1efd563\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 239])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 84b5a094-9eb0-4ff8-be58-5c4913059740 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e5181def-6535-4350-b7f4-acf3785cf121 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b77d62e-b462-4293-950d-00a565b722bd (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 40c1b925-76fc-473c-9317-ffcfbf8aef2d (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 154903a9-f4ab-4352-b947-96c5fce154b3 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b5ceaf3-96d0-4f36-882b-965f9acc31c4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: c7378503-c5b8-4eee-91c0-2c15e3708aea (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 090b4de4-d9d1-47aa-96a9-4cc472900139 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: abc9c059-37e5-408c-b8e0-9fdb2f94fcfb (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ced4eec8-4bf4-46b3-843c-91242defe60a (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 08aaade2-f75a-444c-b97a-b4d3bf97b095 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External } } } }, OmicronZoneConfig { id: 04a81970-7129-40c7-b6f5-36219a8ad854 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cdeb736-9e69-46f9-afb0-59255807dbe4 (service), filesystem_pool: Some(ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External } } } }, OmicronZoneConfig { id: 948190c1-2519-4134-a120-8d6900d50730 (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:28.525237499Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:28.525479891Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:28.525531541Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:28.525661662Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:28.525767133Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:28.525783813Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:28.525925954Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:28.526049345Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:28.526104486Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:28.619071831Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:28.619123711Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:28.619130571Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:28.619138525Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:28.619146635Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:28.619152578Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:28.619158766Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:28.619164622Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:28.619172371Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:28.619178452Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:28.619198142Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:28.619204632Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:28.619213342Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:28.619221182Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:28.619227402Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:28.619235502Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:28.619241552Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:28.619247722Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:28.619254022Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:28.619368203Z","hostname":"sock","pid":1574,"uri":"/omicron-zones","method":"PUT","req_id":"6264fb18-4193-4544-8059-bbb0a5754138","remote_addr":"[fd00:1122:3344:101::1]:49408","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\", TimeNotSynchronized), (\"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\", TimeNotSynchronized), (\"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2\", TimeNotSynchronized), (\"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\", TimeNotSynchronized), (\"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\", TimeNotSynchronized), (\"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139\", TimeNotSynchronized), (\"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\", TimeNotSynchronized), (\"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\", TimeNotSynchronized), (\"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\", TimeNotSynchronized), (\"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\", TimeNotSynchronized), (\"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121\", TimeNotSynchronized), (\"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d\", TimeNotSynchronized), (\"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\", TimeNotSynchronized), (\"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\", TimeNotSynchronized), (\"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\", TimeNotSynchronized), (\"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\", TimeNotSynchronized), (\"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\", TimeNotSynchronized), (\"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3\", TimeNotSynchronized), (\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\", TimeNotSynchronized)]","latency_us":94307,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:28.619549294Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"1.078505755s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6264fb18-4193-4544-8059-bbb0a5754138\", \"content-length\": \"124\", \"date\": \"Mon, 18 Nov 2024 20:59:28 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6264fb18-4193-4544-8059-bbb0a5754138\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 62e83095-0557-4ac0-8b12-7429e79e4bb6 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4554ce4c-90f4-4fd2-93bd-086596238122 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: e9e770f2-88d4-4893-909a-79400af2db8b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b99c98c4-f6a4-43cc-a789-4b40c33f469f, kind: Service { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 }, name: Name(\"ntp-5066b4aa-939d-40f6-9e2e-cc4b97bad1b4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 130, 96])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ddac3753-9787-478c-a9ea-d655291375e0 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3eae245b-f0e3-4234-b3b9-d9b4de089123 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5d056c46-671e-47fd-ab76-2549bb8d565b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: e786554c-6b77-4063-92f2-2aeb18d93f3d (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 14e637a2-51ce-4cac-b831-85b757e5c55d (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 95dce781-7d6b-4bf1-88f1-880b60d0adba, kind: Service { id: 14e637a2-51ce-4cac-b831-85b757e5c55d }, name: Name(\"external-dns-14e637a2-51ce-4cac-b831-85b757e5c55d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 238])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3b3dea8a-e84f-4acf-bafc-3b0e4a815dc5, kind: Service { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 }, name: Name(\"external-dns-be35e232-dd9b-4d59-b4e1-eadb7f69bb77\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 109])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: db0c0ea5-0d8f-4f9d-9979-f3989ce962ea, kind: Service { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 }, name: Name(\"nexus-16a7178a-dbb6-4392-b395-7bddb3817d13\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9b2d2174-b236-4546-bace-300e05afe3c2 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 5dbd0a76-35d9-4b29-8800-824968d1caba, kind: Service { id: 9b2d2174-b236-4546-bace-300e05afe3c2 }, name: Name(\"nexus-9b2d2174-b236-4546-bace-300e05afe3c2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 121])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e67bea26-1a67-4166-875f-3cab431644e6, kind: Service { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 }, name: Name(\"nexus-3dce42ed-74cb-4c8b-9326-8273a1efd563\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 239])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 84b5a094-9eb0-4ff8-be58-5c4913059740 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e5181def-6535-4350-b7f4-acf3785cf121 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b77d62e-b462-4293-950d-00a565b722bd (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 40c1b925-76fc-473c-9317-ffcfbf8aef2d (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 154903a9-f4ab-4352-b947-96c5fce154b3 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b5ceaf3-96d0-4f36-882b-965f9acc31c4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: c7378503-c5b8-4eee-91c0-2c15e3708aea (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 090b4de4-d9d1-47aa-96a9-4cc472900139 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: abc9c059-37e5-408c-b8e0-9fdb2f94fcfb (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ced4eec8-4bf4-46b3-843c-91242defe60a (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 08aaade2-f75a-444c-b97a-b4d3bf97b095 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External } } } }, OmicronZoneConfig { id: 04a81970-7129-40c7-b6f5-36219a8ad854 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cdeb736-9e69-46f9-afb0-59255807dbe4 (service), filesystem_pool: Some(ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External } } } }, OmicronZoneConfig { id: 948190c1-2519-4134-a120-8d6900d50730 (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:29.695880704Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:29.696230345Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:29.696269666Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:29.696407838Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:29.696537068Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:29.696553099Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:29.69670349Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:29.696834651Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:29.696896931Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:29.793844566Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:29.793898127Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:29.793915577Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:29.793921427Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:29.793928637Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:29.793934397Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:29.793939737Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:29.793946347Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:29.793951987Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:29.793957137Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:29.793962125Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:29.793967122Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:29.793973291Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:29.793978128Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:29.793982816Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:29.793986181Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:29.793989537Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:29.794001897Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:29.794006937Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:29.794078968Z","hostname":"sock","pid":1574,"uri":"/omicron-zones","method":"PUT","req_id":"66a06735-8c19-4871-beb8-739c4ae3355a","remote_addr":"[fd00:1122:3344:101::1]:49408","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\", TimeNotSynchronized), (\"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\", TimeNotSynchronized), (\"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3\", TimeNotSynchronized), (\"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2\", TimeNotSynchronized), (\"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\", TimeNotSynchronized), (\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\", TimeNotSynchronized), (\"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\", TimeNotSynchronized), (\"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139\", TimeNotSynchronized), (\"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\", TimeNotSynchronized), (\"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\", TimeNotSynchronized), (\"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d\", TimeNotSynchronized), (\"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\", TimeNotSynchronized), (\"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\", TimeNotSynchronized), (\"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\", TimeNotSynchronized), (\"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\", TimeNotSynchronized), (\"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\", TimeNotSynchronized), (\"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\", TimeNotSynchronized), (\"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\", TimeNotSynchronized), (\"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121\", TimeNotSynchronized)]","latency_us":98313,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:29.794255689Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"921.645134ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"66a06735-8c19-4871-beb8-739c4ae3355a\", \"content-length\": \"124\", \"date\": \"Mon, 18 Nov 2024 20:59:29 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"66a06735-8c19-4871-beb8-739c4ae3355a\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 62e83095-0557-4ac0-8b12-7429e79e4bb6 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4554ce4c-90f4-4fd2-93bd-086596238122 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: e9e770f2-88d4-4893-909a-79400af2db8b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b99c98c4-f6a4-43cc-a789-4b40c33f469f, kind: Service { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 }, name: Name(\"ntp-5066b4aa-939d-40f6-9e2e-cc4b97bad1b4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 130, 96])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ddac3753-9787-478c-a9ea-d655291375e0 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3eae245b-f0e3-4234-b3b9-d9b4de089123 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5d056c46-671e-47fd-ab76-2549bb8d565b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: e786554c-6b77-4063-92f2-2aeb18d93f3d (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 14e637a2-51ce-4cac-b831-85b757e5c55d (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 95dce781-7d6b-4bf1-88f1-880b60d0adba, kind: Service { id: 14e637a2-51ce-4cac-b831-85b757e5c55d }, name: Name(\"external-dns-14e637a2-51ce-4cac-b831-85b757e5c55d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 238])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3b3dea8a-e84f-4acf-bafc-3b0e4a815dc5, kind: Service { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 }, name: Name(\"external-dns-be35e232-dd9b-4d59-b4e1-eadb7f69bb77\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 109])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: db0c0ea5-0d8f-4f9d-9979-f3989ce962ea, kind: Service { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 }, name: Name(\"nexus-16a7178a-dbb6-4392-b395-7bddb3817d13\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9b2d2174-b236-4546-bace-300e05afe3c2 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 5dbd0a76-35d9-4b29-8800-824968d1caba, kind: Service { id: 9b2d2174-b236-4546-bace-300e05afe3c2 }, name: Name(\"nexus-9b2d2174-b236-4546-bace-300e05afe3c2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 121])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e67bea26-1a67-4166-875f-3cab431644e6, kind: Service { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 }, name: Name(\"nexus-3dce42ed-74cb-4c8b-9326-8273a1efd563\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 239])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 84b5a094-9eb0-4ff8-be58-5c4913059740 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e5181def-6535-4350-b7f4-acf3785cf121 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b77d62e-b462-4293-950d-00a565b722bd (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 40c1b925-76fc-473c-9317-ffcfbf8aef2d (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 154903a9-f4ab-4352-b947-96c5fce154b3 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b5ceaf3-96d0-4f36-882b-965f9acc31c4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: c7378503-c5b8-4eee-91c0-2c15e3708aea (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 090b4de4-d9d1-47aa-96a9-4cc472900139 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: abc9c059-37e5-408c-b8e0-9fdb2f94fcfb (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ced4eec8-4bf4-46b3-843c-91242defe60a (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 08aaade2-f75a-444c-b97a-b4d3bf97b095 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External } } } }, OmicronZoneConfig { id: 04a81970-7129-40c7-b6f5-36219a8ad854 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cdeb736-9e69-46f9-afb0-59255807dbe4 (service), filesystem_pool: Some(ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External } } } }, OmicronZoneConfig { id: 948190c1-2519-4134-a120-8d6900d50730 (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:30.71441121Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:30.714756663Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:30.714798643Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:30.715003004Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:30.715142056Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:30.715164366Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:30.715325387Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:30.715450138Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:30.715518948Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:30.818473828Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:30.818534669Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:30.818543279Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:30.818550579Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:30.818554459Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:30.818560119Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:30.818565569Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:30.818572369Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:30.818577869Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:30.818583099Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:30.818595829Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:30.818599849Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:30.818603319Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:30.818608669Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:30.81861395Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:30.818619559Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:30.818624959Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:30.818630099Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:30.81863537Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:30.81868319Z","hostname":"sock","pid":1574,"uri":"/omicron-zones","method":"PUT","req_id":"806d803e-8386-4bbb-b794-82adc34868d0","remote_addr":"[fd00:1122:3344:101::1]:49408","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3\", TimeNotSynchronized), (\"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2\", TimeNotSynchronized), (\"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\", TimeNotSynchronized), (\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\", TimeNotSynchronized), (\"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\", TimeNotSynchronized), (\"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121\", TimeNotSynchronized), (\"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\", TimeNotSynchronized), (\"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\", TimeNotSynchronized), (\"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d\", TimeNotSynchronized), (\"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\", TimeNotSynchronized), (\"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\", TimeNotSynchronized), (\"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\", TimeNotSynchronized), (\"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139\", TimeNotSynchronized), (\"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\", TimeNotSynchronized), (\"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\", TimeNotSynchronized), (\"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\", TimeNotSynchronized), (\"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\", TimeNotSynchronized), (\"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\", TimeNotSynchronized), (\"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\", TimeNotSynchronized)]","latency_us":104445,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:30.818870711Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"717.801173ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"806d803e-8386-4bbb-b794-82adc34868d0\", \"content-length\": \"124\", \"date\": \"Mon, 18 Nov 2024 20:59:30 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"806d803e-8386-4bbb-b794-82adc34868d0\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 62e83095-0557-4ac0-8b12-7429e79e4bb6 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4554ce4c-90f4-4fd2-93bd-086596238122 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: e9e770f2-88d4-4893-909a-79400af2db8b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b99c98c4-f6a4-43cc-a789-4b40c33f469f, kind: Service { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 }, name: Name(\"ntp-5066b4aa-939d-40f6-9e2e-cc4b97bad1b4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 130, 96])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ddac3753-9787-478c-a9ea-d655291375e0 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3eae245b-f0e3-4234-b3b9-d9b4de089123 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5d056c46-671e-47fd-ab76-2549bb8d565b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: e786554c-6b77-4063-92f2-2aeb18d93f3d (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 14e637a2-51ce-4cac-b831-85b757e5c55d (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 95dce781-7d6b-4bf1-88f1-880b60d0adba, kind: Service { id: 14e637a2-51ce-4cac-b831-85b757e5c55d }, name: Name(\"external-dns-14e637a2-51ce-4cac-b831-85b757e5c55d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 238])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3b3dea8a-e84f-4acf-bafc-3b0e4a815dc5, kind: Service { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 }, name: Name(\"external-dns-be35e232-dd9b-4d59-b4e1-eadb7f69bb77\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 109])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: db0c0ea5-0d8f-4f9d-9979-f3989ce962ea, kind: Service { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 }, name: Name(\"nexus-16a7178a-dbb6-4392-b395-7bddb3817d13\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9b2d2174-b236-4546-bace-300e05afe3c2 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 5dbd0a76-35d9-4b29-8800-824968d1caba, kind: Service { id: 9b2d2174-b236-4546-bace-300e05afe3c2 }, name: Name(\"nexus-9b2d2174-b236-4546-bace-300e05afe3c2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 121])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e67bea26-1a67-4166-875f-3cab431644e6, kind: Service { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 }, name: Name(\"nexus-3dce42ed-74cb-4c8b-9326-8273a1efd563\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 239])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 84b5a094-9eb0-4ff8-be58-5c4913059740 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e5181def-6535-4350-b7f4-acf3785cf121 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b77d62e-b462-4293-950d-00a565b722bd (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 40c1b925-76fc-473c-9317-ffcfbf8aef2d (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 154903a9-f4ab-4352-b947-96c5fce154b3 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b5ceaf3-96d0-4f36-882b-965f9acc31c4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: c7378503-c5b8-4eee-91c0-2c15e3708aea (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 090b4de4-d9d1-47aa-96a9-4cc472900139 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: abc9c059-37e5-408c-b8e0-9fdb2f94fcfb (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ced4eec8-4bf4-46b3-843c-91242defe60a (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 08aaade2-f75a-444c-b97a-b4d3bf97b095 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External } } } }, OmicronZoneConfig { id: 04a81970-7129-40c7-b6f5-36219a8ad854 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cdeb736-9e69-46f9-afb0-59255807dbe4 (service), filesystem_pool: Some(ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External } } } }, OmicronZoneConfig { id: 948190c1-2519-4134-a120-8d6900d50730 (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:31.536419597Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:31.536731129Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:31.5367763Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:31.537014112Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:31.537178183Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:31.537213923Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:31.537372174Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:31.537507285Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:31.537593636Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:31.637912876Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:31.637980609Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:31.638004572Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:31.6380103Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:31.638016067Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:31.638021437Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:31.638027237Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:31.638032747Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:31.638038337Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:31.638043677Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:31.638049097Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:31.638055847Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:31.638061487Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:31.638067167Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:31.638073927Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:31.638079078Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:31.638084238Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:31.638092838Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:31.638098428Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:31.638203999Z","hostname":"sock","pid":1574,"uri":"/omicron-zones","method":"PUT","req_id":"1b6ec74e-8859-49a3-9427-ad61ebae5ef5","remote_addr":"[fd00:1122:3344:101::1]:49408","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\", TimeNotSynchronized), (\"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121\", TimeNotSynchronized), (\"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3\", TimeNotSynchronized), (\"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\", TimeNotSynchronized), (\"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\", TimeNotSynchronized), (\"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\", TimeNotSynchronized), (\"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\", TimeNotSynchronized), (\"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\", TimeNotSynchronized), (\"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\", TimeNotSynchronized), (\"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\", TimeNotSynchronized), (\"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\", TimeNotSynchronized), (\"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d\", TimeNotSynchronized), (\"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\", TimeNotSynchronized), (\"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\", TimeNotSynchronized), (\"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2\", TimeNotSynchronized), (\"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\", TimeNotSynchronized), (\"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\", TimeNotSynchronized), (\"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139\", TimeNotSynchronized), (\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\", TimeNotSynchronized)]","latency_us":101940,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:31.63838237Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"865.870945ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1b6ec74e-8859-49a3-9427-ad61ebae5ef5\", \"content-length\": \"124\", \"date\": \"Mon, 18 Nov 2024 20:59:30 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1b6ec74e-8859-49a3-9427-ad61ebae5ef5\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 62e83095-0557-4ac0-8b12-7429e79e4bb6 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4554ce4c-90f4-4fd2-93bd-086596238122 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: e9e770f2-88d4-4893-909a-79400af2db8b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b99c98c4-f6a4-43cc-a789-4b40c33f469f, kind: Service { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 }, name: Name(\"ntp-5066b4aa-939d-40f6-9e2e-cc4b97bad1b4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 130, 96])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ddac3753-9787-478c-a9ea-d655291375e0 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3eae245b-f0e3-4234-b3b9-d9b4de089123 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5d056c46-671e-47fd-ab76-2549bb8d565b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: e786554c-6b77-4063-92f2-2aeb18d93f3d (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 14e637a2-51ce-4cac-b831-85b757e5c55d (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 95dce781-7d6b-4bf1-88f1-880b60d0adba, kind: Service { id: 14e637a2-51ce-4cac-b831-85b757e5c55d }, name: Name(\"external-dns-14e637a2-51ce-4cac-b831-85b757e5c55d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 238])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3b3dea8a-e84f-4acf-bafc-3b0e4a815dc5, kind: Service { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 }, name: Name(\"external-dns-be35e232-dd9b-4d59-b4e1-eadb7f69bb77\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 109])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: db0c0ea5-0d8f-4f9d-9979-f3989ce962ea, kind: Service { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 }, name: Name(\"nexus-16a7178a-dbb6-4392-b395-7bddb3817d13\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9b2d2174-b236-4546-bace-300e05afe3c2 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 5dbd0a76-35d9-4b29-8800-824968d1caba, kind: Service { id: 9b2d2174-b236-4546-bace-300e05afe3c2 }, name: Name(\"nexus-9b2d2174-b236-4546-bace-300e05afe3c2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 121])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e67bea26-1a67-4166-875f-3cab431644e6, kind: Service { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 }, name: Name(\"nexus-3dce42ed-74cb-4c8b-9326-8273a1efd563\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 239])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 84b5a094-9eb0-4ff8-be58-5c4913059740 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e5181def-6535-4350-b7f4-acf3785cf121 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b77d62e-b462-4293-950d-00a565b722bd (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 40c1b925-76fc-473c-9317-ffcfbf8aef2d (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 154903a9-f4ab-4352-b947-96c5fce154b3 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b5ceaf3-96d0-4f36-882b-965f9acc31c4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: c7378503-c5b8-4eee-91c0-2c15e3708aea (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 090b4de4-d9d1-47aa-96a9-4cc472900139 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: abc9c059-37e5-408c-b8e0-9fdb2f94fcfb (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ced4eec8-4bf4-46b3-843c-91242defe60a (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 08aaade2-f75a-444c-b97a-b4d3bf97b095 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External } } } }, OmicronZoneConfig { id: 04a81970-7129-40c7-b6f5-36219a8ad854 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cdeb736-9e69-46f9-afb0-59255807dbe4 (service), filesystem_pool: Some(ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External } } } }, OmicronZoneConfig { id: 948190c1-2519-4134-a120-8d6900d50730 (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:32.503033626Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:32.503324488Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:32.503388448Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:32.503532839Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:32.50366024Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:32.503666491Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:32.503808952Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:32.503942082Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:32.503998223Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:32.603540137Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:32.603598558Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:32.603606168Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:32.603613468Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:32.603619258Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:32.603626388Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:32.603631888Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:32.603637508Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:32.603642498Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:32.603649008Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:32.603670088Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:32.603676118Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:32.603682719Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:32.603687928Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:32.603693148Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:32.603698548Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:32.603702099Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:32.603707259Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:32.603710929Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:32.603790159Z","hostname":"sock","pid":1574,"uri":"/omicron-zones","method":"PUT","req_id":"ab714623-52f2-4df4-8cce-d17ec9aef98a","remote_addr":"[fd00:1122:3344:101::1]:49408","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139\", TimeNotSynchronized), (\"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\", TimeNotSynchronized), (\"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3\", TimeNotSynchronized), (\"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\", TimeNotSynchronized), (\"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\", TimeNotSynchronized), (\"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d\", TimeNotSynchronized), (\"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\", TimeNotSynchronized), (\"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\", TimeNotSynchronized), (\"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\", TimeNotSynchronized), (\"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\", TimeNotSynchronized), (\"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\", TimeNotSynchronized), (\"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2\", TimeNotSynchronized), (\"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121\", TimeNotSynchronized), (\"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\", TimeNotSynchronized), (\"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\", TimeNotSynchronized), (\"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\", TimeNotSynchronized), (\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\", TimeNotSynchronized), (\"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\", TimeNotSynchronized), (\"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\", TimeNotSynchronized)]","latency_us":100922,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:32.603959401Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"1.147277314s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ab714623-52f2-4df4-8cce-d17ec9aef98a\", \"content-length\": \"124\", \"date\": \"Mon, 18 Nov 2024 20:59:32 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ab714623-52f2-4df4-8cce-d17ec9aef98a\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 62e83095-0557-4ac0-8b12-7429e79e4bb6 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4554ce4c-90f4-4fd2-93bd-086596238122 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: e9e770f2-88d4-4893-909a-79400af2db8b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b99c98c4-f6a4-43cc-a789-4b40c33f469f, kind: Service { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 }, name: Name(\"ntp-5066b4aa-939d-40f6-9e2e-cc4b97bad1b4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 130, 96])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ddac3753-9787-478c-a9ea-d655291375e0 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3eae245b-f0e3-4234-b3b9-d9b4de089123 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5d056c46-671e-47fd-ab76-2549bb8d565b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: e786554c-6b77-4063-92f2-2aeb18d93f3d (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 14e637a2-51ce-4cac-b831-85b757e5c55d (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 95dce781-7d6b-4bf1-88f1-880b60d0adba, kind: Service { id: 14e637a2-51ce-4cac-b831-85b757e5c55d }, name: Name(\"external-dns-14e637a2-51ce-4cac-b831-85b757e5c55d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 238])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3b3dea8a-e84f-4acf-bafc-3b0e4a815dc5, kind: Service { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 }, name: Name(\"external-dns-be35e232-dd9b-4d59-b4e1-eadb7f69bb77\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 109])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: db0c0ea5-0d8f-4f9d-9979-f3989ce962ea, kind: Service { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 }, name: Name(\"nexus-16a7178a-dbb6-4392-b395-7bddb3817d13\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9b2d2174-b236-4546-bace-300e05afe3c2 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 5dbd0a76-35d9-4b29-8800-824968d1caba, kind: Service { id: 9b2d2174-b236-4546-bace-300e05afe3c2 }, name: Name(\"nexus-9b2d2174-b236-4546-bace-300e05afe3c2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 121])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e67bea26-1a67-4166-875f-3cab431644e6, kind: Service { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 }, name: Name(\"nexus-3dce42ed-74cb-4c8b-9326-8273a1efd563\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 239])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 84b5a094-9eb0-4ff8-be58-5c4913059740 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e5181def-6535-4350-b7f4-acf3785cf121 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b77d62e-b462-4293-950d-00a565b722bd (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 40c1b925-76fc-473c-9317-ffcfbf8aef2d (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 154903a9-f4ab-4352-b947-96c5fce154b3 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b5ceaf3-96d0-4f36-882b-965f9acc31c4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: c7378503-c5b8-4eee-91c0-2c15e3708aea (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 090b4de4-d9d1-47aa-96a9-4cc472900139 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: abc9c059-37e5-408c-b8e0-9fdb2f94fcfb (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ced4eec8-4bf4-46b3-843c-91242defe60a (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 08aaade2-f75a-444c-b97a-b4d3bf97b095 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External } } } }, OmicronZoneConfig { id: 04a81970-7129-40c7-b6f5-36219a8ad854 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cdeb736-9e69-46f9-afb0-59255807dbe4 (service), filesystem_pool: Some(ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External } } } }, OmicronZoneConfig { id: 948190c1-2519-4134-a120-8d6900d50730 (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:33.748840732Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:33.749122724Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:33.749170785Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:33.749296368Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:33.749401736Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:33.749411496Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:33.749552008Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:33.749669988Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:33.749731229Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:33.849629666Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:33.849678966Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:33.849695196Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:33.849702886Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:33.849708226Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:33.849713677Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:33.849720797Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:33.849725087Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:33.849730427Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:33.849735867Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:33.849741617Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:33.849747037Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:33.849754147Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:33.849759727Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:33.849764787Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:33.849770057Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:33.849777157Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:33.849782417Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:33.849798547Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:33.849909188Z","hostname":"sock","pid":1574,"uri":"/omicron-zones","method":"PUT","req_id":"d0d189dc-1126-451a-8b92-58cb9866f6da","remote_addr":"[fd00:1122:3344:101::1]:49408","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\", TimeNotSynchronized), (\"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121\", TimeNotSynchronized), (\"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\", TimeNotSynchronized), (\"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139\", TimeNotSynchronized), (\"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2\", TimeNotSynchronized), (\"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\", TimeNotSynchronized), (\"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\", TimeNotSynchronized), (\"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3\", TimeNotSynchronized), (\"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d\", TimeNotSynchronized), (\"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\", TimeNotSynchronized), (\"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\", TimeNotSynchronized), (\"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\", TimeNotSynchronized), (\"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\", TimeNotSynchronized), (\"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\", TimeNotSynchronized), (\"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\", TimeNotSynchronized), (\"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\", TimeNotSynchronized), (\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\", TimeNotSynchronized), (\"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\", TimeNotSynchronized), (\"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\", TimeNotSynchronized)]","latency_us":101238,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:33.850086659Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"1.827886946s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d0d189dc-1126-451a-8b92-58cb9866f6da\", \"content-length\": \"124\", \"date\": \"Mon, 18 Nov 2024 20:59:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d0d189dc-1126-451a-8b92-58cb9866f6da\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 62e83095-0557-4ac0-8b12-7429e79e4bb6 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4554ce4c-90f4-4fd2-93bd-086596238122 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: e9e770f2-88d4-4893-909a-79400af2db8b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b99c98c4-f6a4-43cc-a789-4b40c33f469f, kind: Service { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 }, name: Name(\"ntp-5066b4aa-939d-40f6-9e2e-cc4b97bad1b4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 130, 96])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ddac3753-9787-478c-a9ea-d655291375e0 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3eae245b-f0e3-4234-b3b9-d9b4de089123 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5d056c46-671e-47fd-ab76-2549bb8d565b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: e786554c-6b77-4063-92f2-2aeb18d93f3d (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 14e637a2-51ce-4cac-b831-85b757e5c55d (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 95dce781-7d6b-4bf1-88f1-880b60d0adba, kind: Service { id: 14e637a2-51ce-4cac-b831-85b757e5c55d }, name: Name(\"external-dns-14e637a2-51ce-4cac-b831-85b757e5c55d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 238])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3b3dea8a-e84f-4acf-bafc-3b0e4a815dc5, kind: Service { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 }, name: Name(\"external-dns-be35e232-dd9b-4d59-b4e1-eadb7f69bb77\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 109])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: db0c0ea5-0d8f-4f9d-9979-f3989ce962ea, kind: Service { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 }, name: Name(\"nexus-16a7178a-dbb6-4392-b395-7bddb3817d13\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9b2d2174-b236-4546-bace-300e05afe3c2 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 5dbd0a76-35d9-4b29-8800-824968d1caba, kind: Service { id: 9b2d2174-b236-4546-bace-300e05afe3c2 }, name: Name(\"nexus-9b2d2174-b236-4546-bace-300e05afe3c2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 121])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e67bea26-1a67-4166-875f-3cab431644e6, kind: Service { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 }, name: Name(\"nexus-3dce42ed-74cb-4c8b-9326-8273a1efd563\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 239])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 84b5a094-9eb0-4ff8-be58-5c4913059740 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e5181def-6535-4350-b7f4-acf3785cf121 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b77d62e-b462-4293-950d-00a565b722bd (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 40c1b925-76fc-473c-9317-ffcfbf8aef2d (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 154903a9-f4ab-4352-b947-96c5fce154b3 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b5ceaf3-96d0-4f36-882b-965f9acc31c4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: c7378503-c5b8-4eee-91c0-2c15e3708aea (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 090b4de4-d9d1-47aa-96a9-4cc472900139 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: abc9c059-37e5-408c-b8e0-9fdb2f94fcfb (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ced4eec8-4bf4-46b3-843c-91242defe60a (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 08aaade2-f75a-444c-b97a-b4d3bf97b095 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External } } } }, OmicronZoneConfig { id: 04a81970-7129-40c7-b6f5-36219a8ad854 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cdeb736-9e69-46f9-afb0-59255807dbe4 (service), filesystem_pool: Some(ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External } } } }, OmicronZoneConfig { id: 948190c1-2519-4134-a120-8d6900d50730 (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:35.673951319Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:35.674242581Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:35.674273451Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:35.674387342Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:35.674484343Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:35.674490453Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:35.674636774Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:35.674759025Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:35.674822665Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:35.77444408Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:35.774504171Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:35.774512581Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:35.774518431Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:35.774523851Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:35.774529281Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:35.774534921Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:35.774540311Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:35.774545551Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:35.774550951Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:35.774566531Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:35.774573651Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:35.774577691Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:35.774581271Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:35.774585131Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:35.774588671Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:35.774592421Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:35.774596111Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:35.774602601Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:35.774661962Z","hostname":"sock","pid":1574,"uri":"/omicron-zones","method":"PUT","req_id":"620abfac-9492-4cd6-8850-4d9744e6ec32","remote_addr":"[fd00:1122:3344:101::1]:49408","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\", TimeNotSynchronized), (\"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2\", TimeNotSynchronized), (\"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\", TimeNotSynchronized), (\"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\", TimeNotSynchronized), (\"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\", TimeNotSynchronized), (\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\", TimeNotSynchronized), (\"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121\", TimeNotSynchronized), (\"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\", TimeNotSynchronized), (\"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\", TimeNotSynchronized), (\"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\", TimeNotSynchronized), (\"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3\", TimeNotSynchronized), (\"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\", TimeNotSynchronized), (\"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\", TimeNotSynchronized), (\"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\", TimeNotSynchronized), (\"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\", TimeNotSynchronized), (\"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\", TimeNotSynchronized), (\"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d\", TimeNotSynchronized), (\"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\", TimeNotSynchronized), (\"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139\", TimeNotSynchronized)]","latency_us":100872,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:35.774829013Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"2.131169521s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"620abfac-9492-4cd6-8850-4d9744e6ec32\", \"content-length\": \"124\", \"date\": \"Mon, 18 Nov 2024 20:59:35 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"620abfac-9492-4cd6-8850-4d9744e6ec32\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 62e83095-0557-4ac0-8b12-7429e79e4bb6 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4554ce4c-90f4-4fd2-93bd-086596238122 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: e9e770f2-88d4-4893-909a-79400af2db8b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b99c98c4-f6a4-43cc-a789-4b40c33f469f, kind: Service { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 }, name: Name(\"ntp-5066b4aa-939d-40f6-9e2e-cc4b97bad1b4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 130, 96])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ddac3753-9787-478c-a9ea-d655291375e0 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3eae245b-f0e3-4234-b3b9-d9b4de089123 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5d056c46-671e-47fd-ab76-2549bb8d565b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: e786554c-6b77-4063-92f2-2aeb18d93f3d (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 14e637a2-51ce-4cac-b831-85b757e5c55d (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 95dce781-7d6b-4bf1-88f1-880b60d0adba, kind: Service { id: 14e637a2-51ce-4cac-b831-85b757e5c55d }, name: Name(\"external-dns-14e637a2-51ce-4cac-b831-85b757e5c55d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 238])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3b3dea8a-e84f-4acf-bafc-3b0e4a815dc5, kind: Service { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 }, name: Name(\"external-dns-be35e232-dd9b-4d59-b4e1-eadb7f69bb77\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 109])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: db0c0ea5-0d8f-4f9d-9979-f3989ce962ea, kind: Service { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 }, name: Name(\"nexus-16a7178a-dbb6-4392-b395-7bddb3817d13\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9b2d2174-b236-4546-bace-300e05afe3c2 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 5dbd0a76-35d9-4b29-8800-824968d1caba, kind: Service { id: 9b2d2174-b236-4546-bace-300e05afe3c2 }, name: Name(\"nexus-9b2d2174-b236-4546-bace-300e05afe3c2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 121])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e67bea26-1a67-4166-875f-3cab431644e6, kind: Service { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 }, name: Name(\"nexus-3dce42ed-74cb-4c8b-9326-8273a1efd563\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 239])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 84b5a094-9eb0-4ff8-be58-5c4913059740 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e5181def-6535-4350-b7f4-acf3785cf121 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b77d62e-b462-4293-950d-00a565b722bd (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 40c1b925-76fc-473c-9317-ffcfbf8aef2d (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 154903a9-f4ab-4352-b947-96c5fce154b3 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b5ceaf3-96d0-4f36-882b-965f9acc31c4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: c7378503-c5b8-4eee-91c0-2c15e3708aea (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 090b4de4-d9d1-47aa-96a9-4cc472900139 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: abc9c059-37e5-408c-b8e0-9fdb2f94fcfb (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ced4eec8-4bf4-46b3-843c-91242defe60a (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 08aaade2-f75a-444c-b97a-b4d3bf97b095 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External } } } }, OmicronZoneConfig { id: 04a81970-7129-40c7-b6f5-36219a8ad854 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cdeb736-9e69-46f9-afb0-59255807dbe4 (service), filesystem_pool: Some(ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External } } } }, OmicronZoneConfig { id: 948190c1-2519-4134-a120-8d6900d50730 (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:37.902144351Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:37.902419083Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:37.902463113Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:37.902599185Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:37.902713415Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:37.902729026Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:37.903063108Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:37.903207719Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:37.90326721Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:38.003393088Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:38.003456189Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:38.003479129Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:38.003485069Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:38.003491799Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:38.003497689Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:38.003502899Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:38.003508319Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:38.003513289Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:38.003518169Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:38.003523569Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:38.003530149Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:38.00353552Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:38.00354114Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:38.00354814Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:38.003552009Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:38.00355561Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:38.00356071Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:38.00356784Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:38.00364026Z","hostname":"sock","pid":1574,"uri":"/omicron-zones","method":"PUT","req_id":"35ae9ae9-9a45-463b-bda4-b3c5dc360117","remote_addr":"[fd00:1122:3344:101::1]:49408","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2\", TimeNotSynchronized), (\"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\", TimeNotSynchronized), (\"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121\", TimeNotSynchronized), (\"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\", TimeNotSynchronized), (\"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\", TimeNotSynchronized), (\"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139\", TimeNotSynchronized), (\"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\", TimeNotSynchronized), (\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\", TimeNotSynchronized), (\"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\", TimeNotSynchronized), (\"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\", TimeNotSynchronized), (\"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\", TimeNotSynchronized), (\"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3\", TimeNotSynchronized), (\"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\", TimeNotSynchronized), (\"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\", TimeNotSynchronized), (\"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\", TimeNotSynchronized), (\"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\", TimeNotSynchronized), (\"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\", TimeNotSynchronized), (\"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d\", TimeNotSynchronized), (\"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\", TimeNotSynchronized)]","latency_us":101680,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:38.003804691Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"2.896271397s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"35ae9ae9-9a45-463b-bda4-b3c5dc360117\", \"content-length\": \"124\", \"date\": \"Mon, 18 Nov 2024 20:59:37 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"35ae9ae9-9a45-463b-bda4-b3c5dc360117\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:40.158839457Z","hostname":"sock","pid":1574,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"681.208877ms"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 62e83095-0557-4ac0-8b12-7429e79e4bb6 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4554ce4c-90f4-4fd2-93bd-086596238122 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: e9e770f2-88d4-4893-909a-79400af2db8b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b99c98c4-f6a4-43cc-a789-4b40c33f469f, kind: Service { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 }, name: Name(\"ntp-5066b4aa-939d-40f6-9e2e-cc4b97bad1b4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 130, 96])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ddac3753-9787-478c-a9ea-d655291375e0 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3eae245b-f0e3-4234-b3b9-d9b4de089123 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5d056c46-671e-47fd-ab76-2549bb8d565b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: e786554c-6b77-4063-92f2-2aeb18d93f3d (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 14e637a2-51ce-4cac-b831-85b757e5c55d (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 95dce781-7d6b-4bf1-88f1-880b60d0adba, kind: Service { id: 14e637a2-51ce-4cac-b831-85b757e5c55d }, name: Name(\"external-dns-14e637a2-51ce-4cac-b831-85b757e5c55d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 238])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3b3dea8a-e84f-4acf-bafc-3b0e4a815dc5, kind: Service { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 }, name: Name(\"external-dns-be35e232-dd9b-4d59-b4e1-eadb7f69bb77\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 109])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: db0c0ea5-0d8f-4f9d-9979-f3989ce962ea, kind: Service { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 }, name: Name(\"nexus-16a7178a-dbb6-4392-b395-7bddb3817d13\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9b2d2174-b236-4546-bace-300e05afe3c2 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 5dbd0a76-35d9-4b29-8800-824968d1caba, kind: Service { id: 9b2d2174-b236-4546-bace-300e05afe3c2 }, name: Name(\"nexus-9b2d2174-b236-4546-bace-300e05afe3c2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 121])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e67bea26-1a67-4166-875f-3cab431644e6, kind: Service { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 }, name: Name(\"nexus-3dce42ed-74cb-4c8b-9326-8273a1efd563\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 239])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 84b5a094-9eb0-4ff8-be58-5c4913059740 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e5181def-6535-4350-b7f4-acf3785cf121 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b77d62e-b462-4293-950d-00a565b722bd (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 40c1b925-76fc-473c-9317-ffcfbf8aef2d (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 154903a9-f4ab-4352-b947-96c5fce154b3 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b5ceaf3-96d0-4f36-882b-965f9acc31c4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: c7378503-c5b8-4eee-91c0-2c15e3708aea (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 090b4de4-d9d1-47aa-96a9-4cc472900139 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: abc9c059-37e5-408c-b8e0-9fdb2f94fcfb (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ced4eec8-4bf4-46b3-843c-91242defe60a (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 08aaade2-f75a-444c-b97a-b4d3bf97b095 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External } } } }, OmicronZoneConfig { id: 04a81970-7129-40c7-b6f5-36219a8ad854 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cdeb736-9e69-46f9-afb0-59255807dbe4 (service), filesystem_pool: Some(ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External } } } }, OmicronZoneConfig { id: 948190c1-2519-4134-a120-8d6900d50730 (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:40.893701293Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:40.893952014Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:40.893975404Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:40.894090515Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:40.894174116Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:40.894179846Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:40.894306027Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:40.894412648Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:40.894460718Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:40.999939657Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:41.000005828Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:41.000012717Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:41.000019657Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:41.000024777Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:41.000030328Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:41.000035808Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:41.000040988Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:41.000059858Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:41.000065328Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:41.000070448Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:41.000076778Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:41.000082318Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:41.000085898Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:41.000089768Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:41.000095098Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:41.000098918Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:41.000104158Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:41.000107868Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:41.000122818Z","hostname":"sock","pid":1574,"uri":"/omicron-zones","method":"PUT","req_id":"3941552d-3065-4e8a-8e6f-f3b7bb965667","remote_addr":"[fd00:1122:3344:101::1]:49408","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\", TimeNotSynchronized), (\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\", TimeNotSynchronized), (\"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\", TimeNotSynchronized), (\"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\", TimeNotSynchronized), (\"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\", TimeNotSynchronized), (\"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\", TimeNotSynchronized), (\"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\", TimeNotSynchronized), (\"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2\", TimeNotSynchronized), (\"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\", TimeNotSynchronized), (\"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\", TimeNotSynchronized), (\"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139\", TimeNotSynchronized), (\"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d\", TimeNotSynchronized), (\"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\", TimeNotSynchronized), (\"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121\", TimeNotSynchronized), (\"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3\", TimeNotSynchronized), (\"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\", TimeNotSynchronized), (\"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\", TimeNotSynchronized), (\"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\", TimeNotSynchronized), (\"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\", TimeNotSynchronized)]","latency_us":106634,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:41.00030133Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"4.283196937s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3941552d-3065-4e8a-8e6f-f3b7bb965667\", \"content-length\": \"124\", \"date\": \"Mon, 18 Nov 2024 20:59:40 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3941552d-3065-4e8a-8e6f-f3b7bb965667\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 62e83095-0557-4ac0-8b12-7429e79e4bb6 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4554ce4c-90f4-4fd2-93bd-086596238122 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: e9e770f2-88d4-4893-909a-79400af2db8b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b99c98c4-f6a4-43cc-a789-4b40c33f469f, kind: Service { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 }, name: Name(\"ntp-5066b4aa-939d-40f6-9e2e-cc4b97bad1b4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 130, 96])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ddac3753-9787-478c-a9ea-d655291375e0 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3eae245b-f0e3-4234-b3b9-d9b4de089123 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5d056c46-671e-47fd-ab76-2549bb8d565b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: e786554c-6b77-4063-92f2-2aeb18d93f3d (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 14e637a2-51ce-4cac-b831-85b757e5c55d (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 95dce781-7d6b-4bf1-88f1-880b60d0adba, kind: Service { id: 14e637a2-51ce-4cac-b831-85b757e5c55d }, name: Name(\"external-dns-14e637a2-51ce-4cac-b831-85b757e5c55d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 238])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3b3dea8a-e84f-4acf-bafc-3b0e4a815dc5, kind: Service { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 }, name: Name(\"external-dns-be35e232-dd9b-4d59-b4e1-eadb7f69bb77\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 109])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: db0c0ea5-0d8f-4f9d-9979-f3989ce962ea, kind: Service { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 }, name: Name(\"nexus-16a7178a-dbb6-4392-b395-7bddb3817d13\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9b2d2174-b236-4546-bace-300e05afe3c2 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 5dbd0a76-35d9-4b29-8800-824968d1caba, kind: Service { id: 9b2d2174-b236-4546-bace-300e05afe3c2 }, name: Name(\"nexus-9b2d2174-b236-4546-bace-300e05afe3c2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 121])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e67bea26-1a67-4166-875f-3cab431644e6, kind: Service { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 }, name: Name(\"nexus-3dce42ed-74cb-4c8b-9326-8273a1efd563\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 239])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 84b5a094-9eb0-4ff8-be58-5c4913059740 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e5181def-6535-4350-b7f4-acf3785cf121 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b77d62e-b462-4293-950d-00a565b722bd (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 40c1b925-76fc-473c-9317-ffcfbf8aef2d (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 154903a9-f4ab-4352-b947-96c5fce154b3 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b5ceaf3-96d0-4f36-882b-965f9acc31c4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: c7378503-c5b8-4eee-91c0-2c15e3708aea (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 090b4de4-d9d1-47aa-96a9-4cc472900139 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: abc9c059-37e5-408c-b8e0-9fdb2f94fcfb (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ced4eec8-4bf4-46b3-843c-91242defe60a (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 08aaade2-f75a-444c-b97a-b4d3bf97b095 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External } } } }, OmicronZoneConfig { id: 04a81970-7129-40c7-b6f5-36219a8ad854 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cdeb736-9e69-46f9-afb0-59255807dbe4 (service), filesystem_pool: Some(ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External } } } }, OmicronZoneConfig { id: 948190c1-2519-4134-a120-8d6900d50730 (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:45.272935231Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:45.273182913Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:45.273228844Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:45.273363525Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:45.273474555Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:45.273482775Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:45.273623967Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:45.273750288Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:45.273806188Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:45.36764834Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:45.36769379Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:45.36770017Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:45.36770556Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:45.36771094Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:45.36771652Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:45.36772173Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:45.36772791Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:45.36773335Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:45.367738731Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:45.367744011Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:45.3677494Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:45.367756251Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:45.367761511Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:45.367766751Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:45.367772361Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:45.367792711Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:45.367796731Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:45.367801601Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:45.367867452Z","hostname":"sock","pid":1574,"uri":"/omicron-zones","method":"PUT","req_id":"75fb597e-8d42-4dd7-b5fb-1e8fc5622b8d","remote_addr":"[fd00:1122:3344:101::1]:49408","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\", TimeNotSynchronized), (\"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3\", TimeNotSynchronized), (\"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\", TimeNotSynchronized), (\"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\", TimeNotSynchronized), (\"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\", TimeNotSynchronized), (\"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\", TimeNotSynchronized), (\"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d\", TimeNotSynchronized), (\"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139\", TimeNotSynchronized), (\"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2\", TimeNotSynchronized), (\"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121\", TimeNotSynchronized), (\"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\", TimeNotSynchronized), (\"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\", TimeNotSynchronized), (\"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\", TimeNotSynchronized), (\"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\", TimeNotSynchronized), (\"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\", TimeNotSynchronized), (\"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\", TimeNotSynchronized), (\"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\", TimeNotSynchronized), (\"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\", TimeNotSynchronized), (\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\", TimeNotSynchronized)]","latency_us":95097,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:45.368030903Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"4.785708976s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"75fb597e-8d42-4dd7-b5fb-1e8fc5622b8d\", \"content-length\": \"124\", \"date\": \"Mon, 18 Nov 2024 20:59:45 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"75fb597e-8d42-4dd7-b5fb-1e8fc5622b8d\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 62e83095-0557-4ac0-8b12-7429e79e4bb6 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4554ce4c-90f4-4fd2-93bd-086596238122 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: e9e770f2-88d4-4893-909a-79400af2db8b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b99c98c4-f6a4-43cc-a789-4b40c33f469f, kind: Service { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 }, name: Name(\"ntp-5066b4aa-939d-40f6-9e2e-cc4b97bad1b4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 130, 96])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ddac3753-9787-478c-a9ea-d655291375e0 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3eae245b-f0e3-4234-b3b9-d9b4de089123 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5d056c46-671e-47fd-ab76-2549bb8d565b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: e786554c-6b77-4063-92f2-2aeb18d93f3d (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 14e637a2-51ce-4cac-b831-85b757e5c55d (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 95dce781-7d6b-4bf1-88f1-880b60d0adba, kind: Service { id: 14e637a2-51ce-4cac-b831-85b757e5c55d }, name: Name(\"external-dns-14e637a2-51ce-4cac-b831-85b757e5c55d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 238])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3b3dea8a-e84f-4acf-bafc-3b0e4a815dc5, kind: Service { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 }, name: Name(\"external-dns-be35e232-dd9b-4d59-b4e1-eadb7f69bb77\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 109])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: db0c0ea5-0d8f-4f9d-9979-f3989ce962ea, kind: Service { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 }, name: Name(\"nexus-16a7178a-dbb6-4392-b395-7bddb3817d13\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9b2d2174-b236-4546-bace-300e05afe3c2 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 5dbd0a76-35d9-4b29-8800-824968d1caba, kind: Service { id: 9b2d2174-b236-4546-bace-300e05afe3c2 }, name: Name(\"nexus-9b2d2174-b236-4546-bace-300e05afe3c2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 121])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e67bea26-1a67-4166-875f-3cab431644e6, kind: Service { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 }, name: Name(\"nexus-3dce42ed-74cb-4c8b-9326-8273a1efd563\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 239])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 84b5a094-9eb0-4ff8-be58-5c4913059740 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e5181def-6535-4350-b7f4-acf3785cf121 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b77d62e-b462-4293-950d-00a565b722bd (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 40c1b925-76fc-473c-9317-ffcfbf8aef2d (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 154903a9-f4ab-4352-b947-96c5fce154b3 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b5ceaf3-96d0-4f36-882b-965f9acc31c4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: c7378503-c5b8-4eee-91c0-2c15e3708aea (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 090b4de4-d9d1-47aa-96a9-4cc472900139 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: abc9c059-37e5-408c-b8e0-9fdb2f94fcfb (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ced4eec8-4bf4-46b3-843c-91242defe60a (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 08aaade2-f75a-444c-b97a-b4d3bf97b095 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External } } } }, OmicronZoneConfig { id: 04a81970-7129-40c7-b6f5-36219a8ad854 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cdeb736-9e69-46f9-afb0-59255807dbe4 (service), filesystem_pool: Some(ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External } } } }, OmicronZoneConfig { id: 948190c1-2519-4134-a120-8d6900d50730 (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:50.141949733Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:50.142221295Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:50.142262675Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:50.142405716Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:50.142550867Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:50.142562667Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:50.142729868Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:50.142844139Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:50.142898524Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:50.238323983Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:50.238385954Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:50.238393214Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:50.238399544Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:50.238407694Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:50.238413214Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:50.238419924Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:50.238424414Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:50.238440194Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:50.238448314Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:50.238452684Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:50.238457144Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:50.238463374Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:50.238467884Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:50.238474124Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:50.238480544Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:50.238485235Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:50.238489575Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:50.238494145Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:50.238500405Z","hostname":"sock","pid":1574,"uri":"/omicron-zones","method":"PUT","req_id":"63da6706-50ac-44b8-8ed1-f57a16e1958e","remote_addr":"[fd00:1122:3344:101::1]:49408","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\", TimeNotSynchronized), (\"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\", TimeNotSynchronized), (\"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\", TimeNotSynchronized), (\"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\", TimeNotSynchronized), (\"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2\", TimeNotSynchronized), (\"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\", TimeNotSynchronized), (\"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\", TimeNotSynchronized), (\"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\", TimeNotSynchronized), (\"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\", TimeNotSynchronized), (\"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\", TimeNotSynchronized), (\"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139\", TimeNotSynchronized), (\"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\", TimeNotSynchronized), (\"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121\", TimeNotSynchronized), (\"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\", TimeNotSynchronized), (\"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3\", TimeNotSynchronized), (\"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\", TimeNotSynchronized), (\"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d\", TimeNotSynchronized), (\"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\", TimeNotSynchronized), (\"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\", TimeNotSynchronized)]","latency_us":96741,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:50.2386593Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"3.03549812s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"63da6706-50ac-44b8-8ed1-f57a16e1958e\", \"content-length\": \"124\", \"date\": \"Mon, 18 Nov 2024 20:59:50 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"63da6706-50ac-44b8-8ed1-f57a16e1958e\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 62e83095-0557-4ac0-8b12-7429e79e4bb6 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4554ce4c-90f4-4fd2-93bd-086596238122 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: e9e770f2-88d4-4893-909a-79400af2db8b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b99c98c4-f6a4-43cc-a789-4b40c33f469f, kind: Service { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 }, name: Name(\"ntp-5066b4aa-939d-40f6-9e2e-cc4b97bad1b4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 130, 96])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ddac3753-9787-478c-a9ea-d655291375e0 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3eae245b-f0e3-4234-b3b9-d9b4de089123 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5d056c46-671e-47fd-ab76-2549bb8d565b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: e786554c-6b77-4063-92f2-2aeb18d93f3d (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 14e637a2-51ce-4cac-b831-85b757e5c55d (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 95dce781-7d6b-4bf1-88f1-880b60d0adba, kind: Service { id: 14e637a2-51ce-4cac-b831-85b757e5c55d }, name: Name(\"external-dns-14e637a2-51ce-4cac-b831-85b757e5c55d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 238])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3b3dea8a-e84f-4acf-bafc-3b0e4a815dc5, kind: Service { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 }, name: Name(\"external-dns-be35e232-dd9b-4d59-b4e1-eadb7f69bb77\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 109])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: db0c0ea5-0d8f-4f9d-9979-f3989ce962ea, kind: Service { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 }, name: Name(\"nexus-16a7178a-dbb6-4392-b395-7bddb3817d13\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9b2d2174-b236-4546-bace-300e05afe3c2 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 5dbd0a76-35d9-4b29-8800-824968d1caba, kind: Service { id: 9b2d2174-b236-4546-bace-300e05afe3c2 }, name: Name(\"nexus-9b2d2174-b236-4546-bace-300e05afe3c2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 121])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e67bea26-1a67-4166-875f-3cab431644e6, kind: Service { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 }, name: Name(\"nexus-3dce42ed-74cb-4c8b-9326-8273a1efd563\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 239])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 84b5a094-9eb0-4ff8-be58-5c4913059740 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e5181def-6535-4350-b7f4-acf3785cf121 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b77d62e-b462-4293-950d-00a565b722bd (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 40c1b925-76fc-473c-9317-ffcfbf8aef2d (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 154903a9-f4ab-4352-b947-96c5fce154b3 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b5ceaf3-96d0-4f36-882b-965f9acc31c4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: c7378503-c5b8-4eee-91c0-2c15e3708aea (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 090b4de4-d9d1-47aa-96a9-4cc472900139 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: abc9c059-37e5-408c-b8e0-9fdb2f94fcfb (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ced4eec8-4bf4-46b3-843c-91242defe60a (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 08aaade2-f75a-444c-b97a-b4d3bf97b095 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External } } } }, OmicronZoneConfig { id: 04a81970-7129-40c7-b6f5-36219a8ad854 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cdeb736-9e69-46f9-afb0-59255807dbe4 (service), filesystem_pool: Some(ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External } } } }, OmicronZoneConfig { id: 948190c1-2519-4134-a120-8d6900d50730 (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:53.267796318Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:53.268117551Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:53.268158451Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:53.268569244Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:53.268878626Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:53.268929907Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:53.269209039Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:53.26942561Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:53.269495001Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:53.369034785Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:53.369108816Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:53.369118886Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:53.369127816Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:53.369138286Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:53.369145956Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:53.369150936Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:53.369157296Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:53.369161956Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:53.369167762Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:53.369173837Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:53.369179791Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:53.369186887Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:53.369191406Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:53.369196919Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:53.369200707Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:53.369217147Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:53.369221587Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:53.369225897Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:53.369231547Z","hostname":"sock","pid":1574,"uri":"/omicron-zones","method":"PUT","req_id":"b30af96d-58d0-4fdb-82e5-72eb2a6f1255","remote_addr":"[fd00:1122:3344:101::1]:49408","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139\", TimeNotSynchronized), (\"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\", TimeNotSynchronized), (\"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\", TimeNotSynchronized), (\"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\", TimeNotSynchronized), (\"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121\", TimeNotSynchronized), (\"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d\", TimeNotSynchronized), (\"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\", TimeNotSynchronized), (\"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\", TimeNotSynchronized), (\"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\", TimeNotSynchronized), (\"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\", TimeNotSynchronized), (\"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\", TimeNotSynchronized), (\"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\", TimeNotSynchronized), (\"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\", TimeNotSynchronized), (\"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3\", TimeNotSynchronized), (\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\", TimeNotSynchronized), (\"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2\", TimeNotSynchronized), (\"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\", TimeNotSynchronized), (\"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\", TimeNotSynchronized), (\"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\", TimeNotSynchronized)]","latency_us":101631,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:53.369393678Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"4.15134599s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b30af96d-58d0-4fdb-82e5-72eb2a6f1255\", \"content-length\": \"124\", \"date\": \"Mon, 18 Nov 2024 20:59:53 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b30af96d-58d0-4fdb-82e5-72eb2a6f1255\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:54.436833741Z","hostname":"sock","pid":1574,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:61760"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:54.436908481Z","hostname":"sock","pid":1574,"uri":"/baseboard","method":"GET","req_id":"dffe15ea-cb32-4cb2-a174-bd5c6b7a4653","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:61760","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":25,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:55.832304536Z","hostname":"sock","pid":1574,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"2.378630893s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 62e83095-0557-4ac0-8b12-7429e79e4bb6 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4554ce4c-90f4-4fd2-93bd-086596238122 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: e9e770f2-88d4-4893-909a-79400af2db8b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b99c98c4-f6a4-43cc-a789-4b40c33f469f, kind: Service { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 }, name: Name(\"ntp-5066b4aa-939d-40f6-9e2e-cc4b97bad1b4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 130, 96])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ddac3753-9787-478c-a9ea-d655291375e0 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3eae245b-f0e3-4234-b3b9-d9b4de089123 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5d056c46-671e-47fd-ab76-2549bb8d565b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: e786554c-6b77-4063-92f2-2aeb18d93f3d (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 14e637a2-51ce-4cac-b831-85b757e5c55d (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 95dce781-7d6b-4bf1-88f1-880b60d0adba, kind: Service { id: 14e637a2-51ce-4cac-b831-85b757e5c55d }, name: Name(\"external-dns-14e637a2-51ce-4cac-b831-85b757e5c55d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 238])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3b3dea8a-e84f-4acf-bafc-3b0e4a815dc5, kind: Service { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 }, name: Name(\"external-dns-be35e232-dd9b-4d59-b4e1-eadb7f69bb77\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 109])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: db0c0ea5-0d8f-4f9d-9979-f3989ce962ea, kind: Service { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 }, name: Name(\"nexus-16a7178a-dbb6-4392-b395-7bddb3817d13\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9b2d2174-b236-4546-bace-300e05afe3c2 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 5dbd0a76-35d9-4b29-8800-824968d1caba, kind: Service { id: 9b2d2174-b236-4546-bace-300e05afe3c2 }, name: Name(\"nexus-9b2d2174-b236-4546-bace-300e05afe3c2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 121])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e67bea26-1a67-4166-875f-3cab431644e6, kind: Service { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 }, name: Name(\"nexus-3dce42ed-74cb-4c8b-9326-8273a1efd563\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 239])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 84b5a094-9eb0-4ff8-be58-5c4913059740 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e5181def-6535-4350-b7f4-acf3785cf121 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b77d62e-b462-4293-950d-00a565b722bd (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 40c1b925-76fc-473c-9317-ffcfbf8aef2d (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 154903a9-f4ab-4352-b947-96c5fce154b3 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b5ceaf3-96d0-4f36-882b-965f9acc31c4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: c7378503-c5b8-4eee-91c0-2c15e3708aea (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 090b4de4-d9d1-47aa-96a9-4cc472900139 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: abc9c059-37e5-408c-b8e0-9fdb2f94fcfb (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ced4eec8-4bf4-46b3-843c-91242defe60a (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 08aaade2-f75a-444c-b97a-b4d3bf97b095 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External } } } }, OmicronZoneConfig { id: 04a81970-7129-40c7-b6f5-36219a8ad854 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cdeb736-9e69-46f9-afb0-59255807dbe4 (service), filesystem_pool: Some(ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External } } } }, OmicronZoneConfig { id: 948190c1-2519-4134-a120-8d6900d50730 (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:57.510397215Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:57.510669857Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:57.510693547Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:57.510819988Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:57.510916929Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:57.510923829Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:57.51105712Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:57.511171041Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:57.511219602Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:57.610484724Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:57.610558384Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:57.610571605Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:57.610600715Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:57.610610625Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:57.610622425Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:57.610629835Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:57.610638615Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:57.610648175Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:57.610655575Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:57.610665495Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:57.610675035Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:57.610684235Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:57.610697172Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:57.610703472Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:57.61071084Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:57.610716484Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:57.610720685Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:57.610729891Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T20:59:57.610797406Z","hostname":"sock","pid":1574,"uri":"/omicron-zones","method":"PUT","req_id":"35990520-f35c-4cf1-94bb-c4bf618cc30f","remote_addr":"[fd00:1122:3344:101::1]:49408","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\", TimeNotSynchronized), (\"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\", TimeNotSynchronized), (\"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139\", TimeNotSynchronized), (\"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\", TimeNotSynchronized), (\"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\", TimeNotSynchronized), (\"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d\", TimeNotSynchronized), (\"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\", TimeNotSynchronized), (\"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121\", TimeNotSynchronized), (\"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\", TimeNotSynchronized), (\"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3\", TimeNotSynchronized), (\"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\", TimeNotSynchronized), (\"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\", TimeNotSynchronized), (\"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\", TimeNotSynchronized), (\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\", TimeNotSynchronized), (\"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\", TimeNotSynchronized), (\"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2\", TimeNotSynchronized), (\"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\", TimeNotSynchronized), (\"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\", TimeNotSynchronized), (\"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\", TimeNotSynchronized)]","latency_us":100576,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T20:59:57.610953317Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"4.983259043s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"35990520-f35c-4cf1-94bb-c4bf618cc30f\", \"content-length\": \"124\", \"date\": \"Mon, 18 Nov 2024 20:59:57 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"35990520-f35c-4cf1-94bb-c4bf618cc30f\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 62e83095-0557-4ac0-8b12-7429e79e4bb6 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4554ce4c-90f4-4fd2-93bd-086596238122 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: e9e770f2-88d4-4893-909a-79400af2db8b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b99c98c4-f6a4-43cc-a789-4b40c33f469f, kind: Service { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 }, name: Name(\"ntp-5066b4aa-939d-40f6-9e2e-cc4b97bad1b4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 130, 96])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ddac3753-9787-478c-a9ea-d655291375e0 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3eae245b-f0e3-4234-b3b9-d9b4de089123 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5d056c46-671e-47fd-ab76-2549bb8d565b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: e786554c-6b77-4063-92f2-2aeb18d93f3d (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 14e637a2-51ce-4cac-b831-85b757e5c55d (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 95dce781-7d6b-4bf1-88f1-880b60d0adba, kind: Service { id: 14e637a2-51ce-4cac-b831-85b757e5c55d }, name: Name(\"external-dns-14e637a2-51ce-4cac-b831-85b757e5c55d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 238])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3b3dea8a-e84f-4acf-bafc-3b0e4a815dc5, kind: Service { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 }, name: Name(\"external-dns-be35e232-dd9b-4d59-b4e1-eadb7f69bb77\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 109])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: db0c0ea5-0d8f-4f9d-9979-f3989ce962ea, kind: Service { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 }, name: Name(\"nexus-16a7178a-dbb6-4392-b395-7bddb3817d13\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9b2d2174-b236-4546-bace-300e05afe3c2 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 5dbd0a76-35d9-4b29-8800-824968d1caba, kind: Service { id: 9b2d2174-b236-4546-bace-300e05afe3c2 }, name: Name(\"nexus-9b2d2174-b236-4546-bace-300e05afe3c2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 121])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e67bea26-1a67-4166-875f-3cab431644e6, kind: Service { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 }, name: Name(\"nexus-3dce42ed-74cb-4c8b-9326-8273a1efd563\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 239])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 84b5a094-9eb0-4ff8-be58-5c4913059740 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e5181def-6535-4350-b7f4-acf3785cf121 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b77d62e-b462-4293-950d-00a565b722bd (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 40c1b925-76fc-473c-9317-ffcfbf8aef2d (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 154903a9-f4ab-4352-b947-96c5fce154b3 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b5ceaf3-96d0-4f36-882b-965f9acc31c4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: c7378503-c5b8-4eee-91c0-2c15e3708aea (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 090b4de4-d9d1-47aa-96a9-4cc472900139 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: abc9c059-37e5-408c-b8e0-9fdb2f94fcfb (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ced4eec8-4bf4-46b3-843c-91242defe60a (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 08aaade2-f75a-444c-b97a-b4d3bf97b095 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External } } } }, OmicronZoneConfig { id: 04a81970-7129-40c7-b6f5-36219a8ad854 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cdeb736-9e69-46f9-afb0-59255807dbe4 (service), filesystem_pool: Some(ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External } } } }, OmicronZoneConfig { id: 948190c1-2519-4134-a120-8d6900d50730 (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:02.581588739Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:02.582212604Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:02.582245959Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:02.582374405Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:02.582472115Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:02.582498666Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:02.582636887Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:02.582758577Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:02.582812338Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:02.682229492Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:02.682305772Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:02.682317272Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:02.682326682Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:02.682334362Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:02.682343472Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:02.682352913Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:02.682361902Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:02.682367602Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:02.682376322Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:02.682385673Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:02.682418403Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:02.682428413Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:02.682437843Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:02.682445553Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:02.682452873Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:02.682462423Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:02.682471873Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:02.682481263Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:02.682563414Z","hostname":"sock","pid":1574,"uri":"/omicron-zones","method":"PUT","req_id":"92617329-8865-42f2-be1a-a36882a203e9","remote_addr":"[fd00:1122:3344:101::1]:49408","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\", TimeNotSynchronized), (\"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\", TimeNotSynchronized), (\"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139\", TimeNotSynchronized), (\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\", TimeNotSynchronized), (\"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\", TimeNotSynchronized), (\"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\", TimeNotSynchronized), (\"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\", TimeNotSynchronized), (\"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\", TimeNotSynchronized), (\"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\", TimeNotSynchronized), (\"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\", TimeNotSynchronized), (\"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d\", TimeNotSynchronized), (\"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\", TimeNotSynchronized), (\"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2\", TimeNotSynchronized), (\"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\", TimeNotSynchronized), (\"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121\", TimeNotSynchronized), (\"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\", TimeNotSynchronized), (\"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\", TimeNotSynchronized), (\"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\", TimeNotSynchronized), (\"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3\", TimeNotSynchronized)]","latency_us":101144,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:02.682743315Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"7.025557637s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"92617329-8865-42f2-be1a-a36882a203e9\", \"content-length\": \"124\", \"date\": \"Mon, 18 Nov 2024 21:00:02 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"92617329-8865-42f2-be1a-a36882a203e9\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 62e83095-0557-4ac0-8b12-7429e79e4bb6 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4554ce4c-90f4-4fd2-93bd-086596238122 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: e9e770f2-88d4-4893-909a-79400af2db8b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b99c98c4-f6a4-43cc-a789-4b40c33f469f, kind: Service { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 }, name: Name(\"ntp-5066b4aa-939d-40f6-9e2e-cc4b97bad1b4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 130, 96])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ddac3753-9787-478c-a9ea-d655291375e0 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3eae245b-f0e3-4234-b3b9-d9b4de089123 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5d056c46-671e-47fd-ab76-2549bb8d565b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: e786554c-6b77-4063-92f2-2aeb18d93f3d (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 14e637a2-51ce-4cac-b831-85b757e5c55d (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 95dce781-7d6b-4bf1-88f1-880b60d0adba, kind: Service { id: 14e637a2-51ce-4cac-b831-85b757e5c55d }, name: Name(\"external-dns-14e637a2-51ce-4cac-b831-85b757e5c55d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 238])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3b3dea8a-e84f-4acf-bafc-3b0e4a815dc5, kind: Service { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 }, name: Name(\"external-dns-be35e232-dd9b-4d59-b4e1-eadb7f69bb77\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 109])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: db0c0ea5-0d8f-4f9d-9979-f3989ce962ea, kind: Service { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 }, name: Name(\"nexus-16a7178a-dbb6-4392-b395-7bddb3817d13\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9b2d2174-b236-4546-bace-300e05afe3c2 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 5dbd0a76-35d9-4b29-8800-824968d1caba, kind: Service { id: 9b2d2174-b236-4546-bace-300e05afe3c2 }, name: Name(\"nexus-9b2d2174-b236-4546-bace-300e05afe3c2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 121])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e67bea26-1a67-4166-875f-3cab431644e6, kind: Service { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 }, name: Name(\"nexus-3dce42ed-74cb-4c8b-9326-8273a1efd563\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 239])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 84b5a094-9eb0-4ff8-be58-5c4913059740 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e5181def-6535-4350-b7f4-acf3785cf121 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b77d62e-b462-4293-950d-00a565b722bd (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 40c1b925-76fc-473c-9317-ffcfbf8aef2d (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 154903a9-f4ab-4352-b947-96c5fce154b3 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b5ceaf3-96d0-4f36-882b-965f9acc31c4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: c7378503-c5b8-4eee-91c0-2c15e3708aea (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 090b4de4-d9d1-47aa-96a9-4cc472900139 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: abc9c059-37e5-408c-b8e0-9fdb2f94fcfb (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ced4eec8-4bf4-46b3-843c-91242defe60a (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 08aaade2-f75a-444c-b97a-b4d3bf97b095 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External } } } }, OmicronZoneConfig { id: 04a81970-7129-40c7-b6f5-36219a8ad854 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cdeb736-9e69-46f9-afb0-59255807dbe4 (service), filesystem_pool: Some(ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External } } } }, OmicronZoneConfig { id: 948190c1-2519-4134-a120-8d6900d50730 (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:09.690648682Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:09.690937734Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:09.690978644Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:09.691206936Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:09.691345047Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:09.691361197Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:09.691515348Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:09.691620979Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:09.69169055Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:09.800420492Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:09.800484463Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:09.800491603Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:09.800510263Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:09.800517463Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:09.800523073Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:09.800529053Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:09.800534743Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:09.800540403Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:09.800547503Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:09.800553393Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:09.800558633Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:09.800564043Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:09.800569404Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:09.800576504Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:09.800581764Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:09.800587344Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:09.800594024Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:09.800610354Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3321","zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:09.800720464Z","hostname":"sock","pid":1574,"uri":"/omicron-zones","method":"PUT","req_id":"9d5489e4-69cb-479b-bd25-1a9f27301bb5","remote_addr":"[fd00:1122:3344:101::1]:49408","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:851","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\", TimeNotSynchronized), (\"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\", TimeNotSynchronized), (\"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121\", TimeNotSynchronized), (\"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d\", TimeNotSynchronized), (\"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3\", TimeNotSynchronized), (\"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139\", TimeNotSynchronized), (\"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\", TimeNotSynchronized), (\"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\", TimeNotSynchronized), (\"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\", TimeNotSynchronized), (\"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\", TimeNotSynchronized), (\"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\", TimeNotSynchronized), (\"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\", TimeNotSynchronized), (\"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\", TimeNotSynchronized), (\"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\", TimeNotSynchronized), (\"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2\", TimeNotSynchronized), (\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\", TimeNotSynchronized), (\"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\", TimeNotSynchronized), (\"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\", TimeNotSynchronized), (\"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\", TimeNotSynchronized)]","latency_us":110251,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:09.800887766Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:575","retry_after":"8.030578433s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9d5489e4-69cb-479b-bd25-1a9f27301bb5\", \"content-length\": \"124\", \"date\": \"Mon, 18 Nov 2024 21:00:09 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9d5489e4-69cb-479b-bd25-1a9f27301bb5\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:13.202438983Z","hostname":"sock","pid":1574,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"4.428802477s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 62e83095-0557-4ac0-8b12-7429e79e4bb6 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 4554ce4c-90f4-4fd2-93bd-086596238122 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: e9e770f2-88d4-4893-909a-79400af2db8b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: b99c98c4-f6a4-43cc-a789-4b40c33f469f, kind: Service { id: 5066b4aa-939d-40f6-9e2e-cc4b97bad1b4 }, name: Name(\"ntp-5066b4aa-939d-40f6-9e2e-cc4b97bad1b4\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 130, 96])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: ddac3753-9787-478c-a9ea-d655291375e0 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3eae245b-f0e3-4234-b3b9-d9b4de089123 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 5d056c46-671e-47fd-ab76-2549bb8d565b (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: e786554c-6b77-4063-92f2-2aeb18d93f3d (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cf5dd9c-cf98-4b6d-b813-c89d2e1c27ce (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 14e637a2-51ce-4cac-b831-85b757e5c55d (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 95dce781-7d6b-4bf1-88f1-880b60d0adba, kind: Service { id: 14e637a2-51ce-4cac-b831-85b757e5c55d }, name: Name(\"external-dns-14e637a2-51ce-4cac-b831-85b757e5c55d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 238])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 3b3dea8a-e84f-4acf-bafc-3b0e4a815dc5, kind: Service { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 }, name: Name(\"external-dns-be35e232-dd9b-4d59-b4e1-eadb7f69bb77\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 109])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: db0c0ea5-0d8f-4f9d-9979-f3989ce962ea, kind: Service { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 }, name: Name(\"nexus-16a7178a-dbb6-4392-b395-7bddb3817d13\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9b2d2174-b236-4546-bace-300e05afe3c2 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 5dbd0a76-35d9-4b29-8800-824968d1caba, kind: Service { id: 9b2d2174-b236-4546-bace-300e05afe3c2 }, name: Name(\"nexus-9b2d2174-b236-4546-bace-300e05afe3c2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 121])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: e67bea26-1a67-4166-875f-3cab431644e6, kind: Service { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 }, name: Name(\"nexus-3dce42ed-74cb-4c8b-9326-8273a1efd563\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 239])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 84b5a094-9eb0-4ff8-be58-5c4913059740 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: e5181def-6535-4350-b7f4-acf3785cf121 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1b77d62e-b462-4293-950d-00a565b722bd (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 40c1b925-76fc-473c-9317-ffcfbf8aef2d (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 154903a9-f4ab-4352-b947-96c5fce154b3 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b5ceaf3-96d0-4f36-882b-965f9acc31c4 (service), filesystem_pool: Some(ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External } } } }, OmicronZoneConfig { id: c7378503-c5b8-4eee-91c0-2c15e3708aea (service), filesystem_pool: Some(ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External } } } }, OmicronZoneConfig { id: 090b4de4-d9d1-47aa-96a9-4cc472900139 (service), filesystem_pool: Some(ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External } } } }, OmicronZoneConfig { id: abc9c059-37e5-408c-b8e0-9fdb2f94fcfb (service), filesystem_pool: Some(ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External } } } }, OmicronZoneConfig { id: ced4eec8-4bf4-46b3-843c-91242defe60a (service), filesystem_pool: Some(ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External } } } }, OmicronZoneConfig { id: 08aaade2-f75a-444c-b97a-b4d3bf97b095 (service), filesystem_pool: Some(ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External } } } }, OmicronZoneConfig { id: 04a81970-7129-40c7-b6f5-36219a8ad854 (service), filesystem_pool: Some(ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6cdeb736-9e69-46f9-afb0-59255807dbe4 (service), filesystem_pool: Some(ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External } } } }, OmicronZoneConfig { id: 948190c1-2519-4134-a120-8d6900d50730 (service), filesystem_pool: Some(ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:17.810367692Z","hostname":"sock","pid":1574,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:538"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:17.810649784Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:17.810681354Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:17.810867716Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsConfigList { tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:17.810993387Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:17.811007337Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-datasets.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:17.811181858Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"common/src/ledger.rs:178"} {"msg":"Found ledger on internal storage","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:17.811317709Z","hostname":"sock","pid":1574,"request":"datasets_config_list","component":"StorageManager","file":"sled-storage/src/manager.rs:992"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:17.81138622Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:17.904741188Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:17.952426984Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:17.952610696Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:17.952658695Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:17.952670732Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:17.952676132Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:17.953881355Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.004643005Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.004683225Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.004687485Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.165422517Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.165641299Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.204135907Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:264"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.204355098Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.279832603Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.279878073Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.279882263Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.279885903Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.279889403Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Configuring new Omicron zone: oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.280621689Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.285911418Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.291091407Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.295847973Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.301607176Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.306650823Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.31158543Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.446973963Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.452128391Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.456951838Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.461391321Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.499185583Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Installing Omicron zone: oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.499378075Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.503763297Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.507831038Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.512536343Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.51747274Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.523039072Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.528536093Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.534651939Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_948190c1-2519-4134-a120-8d6900d50730","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.540783005Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.547540725Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.610523756Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.611201301Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.619943997Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.626792378Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.632695072Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.638961789Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_948190c1-2519-4134-a120-8d6900d50730","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.645564638Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.652722131Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.715580331Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.770117889Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.827785201Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:281","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.830509021Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.830524421Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.886992693Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte1\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 242, 19]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.887025064Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:1271","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.89052128Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.89055841Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.946354397Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte2\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 201, 121]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.946386618Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:1271","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.948173121Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:18.948188001Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:19.00413385Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte3\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 153, 239]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:19.00416244Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:1271","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:19.007481195Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:19.007495424Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:19.071755855Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte4\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 185, 109]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:19.071785985Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:1271","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:19.073519078Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:306"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:19.073532029Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:198","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:19.124997723Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte5\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 153, 238]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:19.125021383Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:1271","zone_type":"external_dns"} {"msg":"no nat entry found for: 10.151.2.166","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:19.125264875Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.166","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:19.125278156Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.2.167","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:19.125354776Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.167","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:19.125360336Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.2.168","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:19.125408166Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.168","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:19.125413637Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.2.166","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:19.129544137Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:19.131093109Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"creation of nat entry successful for: 10.151.2.167","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:19.131121589Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"creation of nat entry successful for: 10.151.2.168","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:19.132811072Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:19.13521118Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:19.13523029Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"no nat entry found for: 10.151.2.165","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:19.1352363Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.165","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:19.13524229Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.2.164","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:19.13526678Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.2.164","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:19.13528939Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.2.165","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:19.258067198Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"creation of nat entry successful for: 10.151.2.164","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:19.260420586Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:19.262768374Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:19.262798724Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Configuring new Omicron zone: oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:19.263198157Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:19.270666343Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:19.355681279Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Configuring new Omicron zone: oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:19.364956798Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:19.37465451Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:19.385995505Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Configuring new Omicron zone: oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:19.399755762Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:19.41063699Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:19.420234351Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"Installing Omicron zone: oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:19.446967251Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:29.847930671Z","hostname":"sock","pid":1574,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:43893"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:29.89062511Z","hostname":"sock","pid":1574,"uri":"/baseboard","method":"GET","req_id":"07b41656-9d42-4eb4-ab48-752f61852423","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:43893","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":455,"response_code":"200"} {"msg":"Profile for oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:30.40529717Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:30.405875844Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:30.423743608Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:30.423991959Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:30.526128583Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:30.526513436Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:30.526567746Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:30.535600583Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:30.535963967Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:30.552772602Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:30.55371148Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:30.554965879Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:30.555018719Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:30.565293546Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_948190c1-2519-4134-a120-8d6900d50730:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:30.566102002Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:30.569711139Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:30.580706421Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:30.592284078Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:30.592352718Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:30.604366548Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_crucible_948190c1-2519-4134-a120-8d6900d50730 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:30.615801084Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:30.628064885Z","hostname":"sock","pid":1574,"zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Profile for oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:30.928059208Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:30.928716953Z","hostname":"sock","pid":1574,"zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\") failed: Property not found. retry in 44.760635ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:31.053518837Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\") failed: Property not found. retry in 57.713305ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:31.065823459Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\") failed: Property not found. retry in 40.310342ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:31.078485964Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\") failed: Property not found. retry in 27.961089ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:31.087894884Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\") failed: Property not found. retry in 29.463489ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:31.096530088Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\") failed: Property not found. retry in 32.08457ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:31.104693089Z","hostname":"sock","pid":1574,"zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\") failed: Property not found. retry in 72.808378ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:31.124342287Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\") failed: Property not found. retry in 62.837352ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:31.137091492Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\") failed: Property not found. retry in 66.598023ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:31.148213895Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\") failed: Property not found. retry in 67.520027ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:31.160335076Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\") failed: Property not found. retry in 60.609831ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:31.172116664Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\") failed: Property not found. retry in 75.391629ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:31.182714043Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\") failed: Property not found. retry in 42.296004ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:31.194611352Z","hostname":"sock","pid":1574,"zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\") failed: Property not found. retry in 69.602485ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:31.206318009Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\") failed: Property not found. retry in 122.326963ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:31.216812778Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\") failed: Property not found. retry in 126.040599ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:31.228966859Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\") failed: Property not found. retry in 68.050581ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:31.238800652Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\") failed: Property not found. retry in 67.629756ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:31.250435999Z","hostname":"sock","pid":1574,"zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\") failed: Property not found. retry in 81.367798ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:31.261474192Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:31.262728601Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\") failed: Property not found. retry in 93.904103ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:32.281904123Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\") failed: Property not found. retry in 118.885599ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:32.305122667Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\") failed: Property not found. retry in 54.508226ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:32.347534634Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\") failed: Property not found. retry in 53.585652ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:32.549574295Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:32.63442092Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\") failed: Property not found. retry in 246.865333ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:32.745673791Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:32.746276476Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\") failed: Property not found. retry in 115.886538ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:32.784106328Z","hostname":"sock","pid":1574,"zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\") failed: Property not found. retry in 132.02695ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:32.877063104Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\") failed: Property not found. retry in 150.773609ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:32.890273862Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\") failed: Property not found. retry in 280.285776ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:32.967204528Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\") failed: Property not found. retry in 289.150859ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:33.025178291Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\") failed: Property not found. retry in 292.797032ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:33.128705523Z","hostname":"sock","pid":1574,"zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\") failed: Property not found. retry in 294.956769ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:33.200564899Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:33.20064549Z","hostname":"sock","pid":1574,"zone":"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:33.280882848Z","hostname":"sock","pid":1574,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"9.939392662s"} {"msg":"wait for service Some(\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\") failed: Property not found. retry in 297.06442ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:33.299702898Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\") failed: Property not found. retry in 193.2948ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:33.311253535Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\") failed: Property not found. retry in 209.608869ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:33.320787166Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\") failed: Property not found. retry in 297.564094ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:33.331707347Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:33.331769708Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\") failed: Property not found. retry in 510.754782ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:33.388142718Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:33.388231649Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\") failed: Property not found. retry in 144.487859ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:33.405427377Z","hostname":"sock","pid":1574,"zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\") failed: Property not found. retry in 347.621404ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:33.414911158Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\") failed: Property not found. retry in 322.22428ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:33.425668578Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\") failed: Property not found. retry in 495.543716ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:33.437556697Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\") failed: Property not found. retry in 234.042628ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:33.44880273Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\") failed: Property not found. retry in 405.387699ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:33.459306559Z","hostname":"sock","pid":1574,"zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\") failed: Property not found. retry in 453.140946ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:33.756727817Z","hostname":"sock","pid":1574,"zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\") failed: Property not found. retry in 1.022992572s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:33.768611235Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\") failed: Property not found. retry in 924.486683ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:33.780364383Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\") failed: Property not found. retry in 200.895303ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:33.789640552Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\") failed: Property not found. retry in 296.248497ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:33.797927074Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\") failed: Property not found. retry in 363.678099ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:33.809164868Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\") failed: Property not found. retry in 388.436004ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:33.818495307Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\") failed: Property not found. retry in 342.003034ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:33.827173022Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\") failed: Property not found. retry in 724.528887ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:33.838539847Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:33.838824959Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:33.8390355Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:33.839241062Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"Booting oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:33.84563696Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\") failed: Property not found. retry in 1.078130936s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:33.876090287Z","hostname":"sock","pid":1574,"zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\") failed: Property not found. retry in 701.382805ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:33.911113078Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\") failed: Property not found. retry in 951.589117ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:33.946449081Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\") failed: Property not found. retry in 1.168595179s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:34.003512667Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\") failed: Property not found. retry in 33.294248ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:34.042634039Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\") failed: Property not found. retry in 58.262198ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:34.053287718Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\") failed: Property not found. retry in 138.622742ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:34.088472551Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\") failed: Property not found. retry in 460.907677ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:34.107713994Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\") failed: Property not found. retry in 68.767309ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:34.124276498Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\") failed: Property not found. retry in 627.700867ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:34.181872947Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\") failed: Property not found. retry in 1.128997292s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:34.192536367Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\") failed: Property not found. retry in 227.887343ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:34.203098976Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\") failed: Property not found. retry in 689.772305ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:34.2198035Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\") failed: Property not found. retry in 1.131200228s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:34.232517175Z","hostname":"sock","pid":1574,"zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\") failed: Property not found. retry in 175.049408ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:34.246918912Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\") failed: Property not found. retry in 562.670349ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:34.439255077Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\") failed: Property not found. retry in 442.256111ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:34.455083035Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\") failed: Property not found. retry in 1.162410212s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:34.57505881Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\") failed: Property not found. retry in 615.351112ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:34.587292401Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\") failed: Property not found. retry in 1.339142839s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:34.62735413Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\") failed: Property not found. retry in 1.433737411s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:34.719381856Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\") failed: Property not found. retry in 1.149328101s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:34.805427678Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\") failed: Property not found. retry in 1.189646765s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:34.822755407Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\") failed: Property not found. retry in 1.148156123s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:34.9103271Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\") failed: Property not found. retry in 944.094655ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:34.920879029Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\") failed: Property not found. retry in 1.481691897s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:34.931480478Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\") failed: Property not found. retry in 1.036194813s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:34.967339155Z","hostname":"sock","pid":1574,"zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\") failed: Property not found. retry in 697.499124ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:35.014731969Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\") failed: Property not found. retry in 1.286025988s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:35.184922028Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\") failed: Property not found. retry in 1.424535574s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:35.21596433Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\") failed: Property not found. retry in 654.20361ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:35.33393746Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\") failed: Property not found. retry in 658.028632ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:35.375399329Z","hostname":"sock","pid":1574,"zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\") failed: Property not found. retry in 1.40426093s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:35.727934858Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\") failed: Property not found. retry in 1.358941375s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:35.74963527Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\") failed: Property not found. retry in 1.167051447s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:35.877689725Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\") failed: Property not found. retry in 1.043692487s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:35.967559265Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\") failed: Property not found. retry in 1.264359614s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:35.976757054Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\") failed: Property not found. retry in 874.684484ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:35.99906684Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\") failed: Property not found. retry in 1.215513226s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:36.015441692Z","hostname":"sock","pid":1574,"zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\") failed: Property not found. retry in 768.820227ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:36.025099524Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\") failed: Property not found. retry in 1.039360827s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:36.043199549Z","hostname":"sock","pid":1574,"zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\") failed: Property not found. retry in 1.452852863s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:36.072202715Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\") failed: Property not found. retry in 1.49580642s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:36.167076263Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\") failed: Property not found. retry in 712.44783ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:36.426395457Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\") failed: Property not found. retry in 1.104549209s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:36.483526963Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\") failed: Property not found. retry in 660.795934ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:36.655626667Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\") failed: Property not found. retry in 1.426147645s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:36.808240935Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\") failed: Property not found. retry in 919.839542ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:36.888821316Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\") failed: Property not found. retry in 1.321344957s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:37.026895835Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\") failed: Property not found. retry in 1.393979023s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:37.059010165Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\") failed: Property not found. retry in 1.34782223s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:37.097161209Z","hostname":"sock","pid":1574,"zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\") failed: Property not found. retry in 580.333531ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:37.121717562Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\") failed: Property not found. retry in 1.381760162s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:37.142491848Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\") failed: Property not found. retry in 1.053576438s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:37.151146442Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\") failed: Property not found. retry in 1.324175213s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:37.24470696Z","hostname":"sock","pid":1574,"zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\") failed: Property not found. retry in 1.093537502s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:37.254887876Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\") failed: Property not found. retry in 663.241406ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:37.326936043Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\") failed: Property not found. retry in 570.009948ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:37.535070325Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\") failed: Property not found. retry in 948.125098ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:37.597860614Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\") failed: Property not found. retry in 1.081440755s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:37.671847036Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\") failed: Property not found. retry in 522.698248ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:37.710973787Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\") failed: Property not found. retry in 1.281731907s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:37.817603152Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\") failed: Property not found. retry in 656.967977ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:38.000200864Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\") failed: Property not found. retry in 845.695182ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:38.113176007Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\") failed: Property not found. retry in 1.177021649s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:38.213701637Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\") failed: Property not found. retry in 1.102498849s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:38.241656965Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\") failed: Property not found. retry in 1.012091248s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:38.249228662Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\") failed: Property not found. retry in 648.109267ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:38.357311857Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\") failed: Property not found. retry in 1.364852697s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:38.364462901Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\") failed: Property not found. retry in 1.3983999s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:38.453671666Z","hostname":"sock","pid":1574,"zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\") failed: Property not found. retry in 1.20042025s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:38.461167562Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\") failed: Property not found. retry in 1.012966507s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:38.533130899Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\") failed: Property not found. retry in 683.892797ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:38.553900194Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\") failed: Property not found. retry in 563.233091ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:38.576815585Z","hostname":"sock","pid":1574,"zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\") failed: Property not found. retry in 833.70103ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:38.665909579Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\") failed: Property not found. retry in 1.019829777s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:38.762170687Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\") failed: Property not found. retry in 1.359598297s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:38.968220044Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\") failed: Property not found. retry in 1.177207761s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:39.014963682Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\") failed: Property not found. retry in 1.153326812s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:39.108113637Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\") failed: Property not found. retry in 966.898753ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:39.148171676Z","hostname":"sock","pid":1574,"zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\") failed: Property not found. retry in 558.794611ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:39.246376668Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\") failed: Property not found. retry in 1.334066176s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:39.270074685Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\") failed: Property not found. retry in 708.852249ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:39.351894615Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\") failed: Property not found. retry in 970.577386ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:39.399602781Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\") failed: Property not found. retry in 567.518491ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:39.508551303Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\") failed: Property not found. retry in 551.051768ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:39.554725968Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\") failed: Property not found. retry in 1.407362263s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:39.70383355Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\") failed: Property not found. retry in 938.287899ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:39.845799649Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\") failed: Property not found. retry in 822.635635ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:39.940122942Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\") failed: Property not found. retry in 1.309982874s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:39.98629322Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\") failed: Property not found. retry in 1.235478304s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:40.032218429Z","hostname":"sock","pid":1574,"zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\") failed: Property not found. retry in 965.380074ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:40.068295778Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\") failed: Property not found. retry in 573.003939ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:40.083956445Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\") failed: Property not found. retry in 1.421366798s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:40.114722644Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\") failed: Property not found. retry in 1.196493266s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:40.122906725Z","hostname":"sock","pid":1574,"zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\") failed: Property not found. retry in 1.229970791s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:40.204313512Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\") failed: Property not found. retry in 539.831917ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:40.271605424Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\") failed: Property not found. retry in 1.125907455s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:40.337457035Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\") failed: Property not found. retry in 935.168078ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:40.379725341Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\") failed: Property not found. retry in 1.264031902s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:40.613249102Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\") failed: Property not found. retry in 1.273508913s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:40.665936575Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\") failed: Property not found. retry in 1.140503832s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:40.771406041Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\") failed: Property not found. retry in 1.301593842s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:40.793532946Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\") failed: Property not found. retry in 980.923828ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:40.821263833Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\") failed: Property not found. retry in 907.727134ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:41.042605354Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\") failed: Property not found. retry in 1.388827299s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:41.121617313Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\") failed: Property not found. retry in 829.465203ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:41.277275374Z","hostname":"sock","pid":1574,"zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\") failed: Property not found. retry in 1.122403922s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:41.306013149Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\") failed: Property not found. retry in 1.358121025s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:41.322954375Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\") failed: Property not found. retry in 1.082141186s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:41.330890614Z","hostname":"sock","pid":1574,"zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\") failed: Property not found. retry in 885.384973ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:41.443429664Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\") failed: Property not found. retry in 1.268120689s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:41.472094618Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\") failed: Property not found. retry in 655.849198ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:41.544629048Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\") failed: Property not found. retry in 1.035756858s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:41.812797688Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\") failed: Property not found. retry in 1.296228265s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:41.887220854Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\") failed: Property not found. retry in 1.345256769s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:41.920945215Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\") failed: Property not found. retry in 712.070991ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:41.949611429Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\") failed: Property not found. retry in 1.463010449s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:41.959514133Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\") failed: Property not found. retry in 1.375584319s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:42.103539577Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\") failed: Property not found. retry in 597.053739ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:42.119070633Z","hostname":"sock","pid":1574,"zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\") failed: Property not found. retry in 740.88252ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:42.210006271Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\") failed: Property not found. retry in 1.475417043s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:42.337817794Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\") failed: Property not found. retry in 1.34802664s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:42.423564034Z","hostname":"sock","pid":1574,"zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\") failed: Property not found. retry in 729.514942ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:42.438425434Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\") failed: Property not found. retry in 870.364428ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:42.519480099Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\") failed: Property not found. retry in 539.047325ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:42.671441002Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\") failed: Property not found. retry in 760.769331ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:42.689971021Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\") failed: Property not found. retry in 1.051237557s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:42.725375584Z","hostname":"sock","pid":1574,"zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\") failed: Property not found. retry in 1.488722254s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:42.749649765Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\") failed: Property not found. retry in 1.152664583s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:42.858359276Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\") failed: Property not found. retry in 943.597549ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:42.960544618Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\") failed: Property not found. retry in 1.270288275s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:43.178371333Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\") failed: Property not found. retry in 1.191076135s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:43.193643246Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\") failed: Property not found. retry in 834.871547ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:43.220683498Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\") failed: Property not found. retry in 768.042268ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:43.426582214Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\") failed: Property not found. retry in 894.082625ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:43.434548823Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\") failed: Property not found. retry in 687.450265ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:43.441712057Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\") failed: Property not found. retry in 609.784932ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:43.460022093Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\") failed: Property not found. retry in 1.023234103s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:43.487874981Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\") failed: Property not found. retry in 1.072498501s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:43.786383887Z","hostname":"sock","pid":1574,"zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\") failed: Property not found. retry in 930.873782ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:43.799363004Z","hostname":"sock","pid":1574,"zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\") failed: Property not found. retry in 1.355032817s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:43.827133491Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\") failed: Property not found. retry in 693.645286ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:43.921728817Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d\") failed: Property not found. retry in 68.431228ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:43.938583722Z","hostname":"sock","pid":1574,"zone":"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3\") failed: Property not found. retry in 54.45448ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:43.9476589Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139\") failed: Property not found. retry in 68.285236ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:43.955511659Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3\") failed: Property not found. retry in 118.100985ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:44.013191909Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d\") failed: Property not found. retry in 140.250389ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:44.023169953Z","hostname":"sock","pid":1574,"zone":"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\") failed: Property not found. retry in 1.040298203s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:44.03344576Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139\") failed: Property not found. retry in 125.042591ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:44.041767732Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\") failed: Property not found. retry in 832.309007ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:44.064412491Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\") failed: Property not found. retry in 1.13299462s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:44.175039936Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3\") failed: Property not found. retry in 125.756384ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:44.18493118Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\") failed: Property not found. retry in 1.073352252s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:44.19435675Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d\") failed: Property not found. retry in 182.342146ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:44.202543921Z","hostname":"sock","pid":1574,"zone":"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139\") failed: Property not found. retry in 101.998086ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:44.210657202Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\") failed: Property not found. retry in 647.631339ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:44.219684299Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\") failed: Property not found. retry in 1.005461035s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:44.248669665Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3\") failed: Property not found. retry in 336.247883ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:44.324113888Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139\") failed: Property not found. retry in 304.296954ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:44.360216997Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\") failed: Property not found. retry in 1.022871573s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:44.385735997Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d\") failed: Property not found. retry in 523.137232ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:44.420077753Z","hostname":"sock","pid":1574,"zone":"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\") failed: Property not found. retry in 1.001618676s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:44.44106941Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\") failed: Property not found. retry in 952.984718ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:44.478469599Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:44.489567452Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:44.489665483Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\") failed: Property not found. retry in 1.038688879s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:44.520495232Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\") failed: Property not found. retry in 1.125915436s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:44.627253429Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3\") failed: Property not found. retry in 602.439662ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:44.670598902Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139\") failed: Property not found. retry in 440.345895ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:44.679352207Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2\") failed: Property not found. retry in 38.403342ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:44.696032282Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\") failed: Property not found. retry in 535.254477ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:44.742271396Z","hostname":"sock","pid":1574,"zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2\") failed: Property not found. retry in 82.464202ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:44.749884433Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2\") failed: Property not found. retry in 132.776143ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:44.864396517Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\") failed: Property not found. retry in 541.804552ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:44.880547447Z","hostname":"sock","pid":1574,"zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\") failed: Property not found. retry in 1.487893327s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:44.894179419Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\") failed: Property not found. retry in 915.282828ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:44.909566284Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d\") failed: Property not found. retry in 779.293306ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:44.955513286Z","hostname":"sock","pid":1574,"zone":"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2\") failed: Property not found. retry in 471.772762ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:45.009690479Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\") failed: Property not found. retry in 1.126473331s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:45.08467571Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139\") failed: Property not found. retry in 746.469655ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:45.130256349Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\") failed: Property not found. retry in 1.024440321s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:45.192582925Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\") failed: Property not found. retry in 820.766216ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:45.263790945Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\") failed: Property not found. retry in 1.267481865s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:45.278448435Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3\") failed: Property not found. retry in 1.342250886s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:45.288361119Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\") failed: Property not found. retry in 1.18675247s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:45.297559157Z","hostname":"sock","pid":1574,"zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\") failed: Property not found. retry in 895.814999ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:45.318887707Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\") failed: Property not found. retry in 817.75233ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:45.419312176Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\") failed: Property not found. retry in 1.300491829s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:45.435152904Z","hostname":"sock","pid":1574,"zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\") failed: Property not found. retry in 780.093006ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:45.444138991Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\") failed: Property not found. retry in 1.320258454s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:45.452785425Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2\") failed: Property not found. retry in 973.109667ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:45.491634895Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\") failed: Property not found. retry in 880.247083ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:45.569537756Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d\") failed: Property not found. retry in 623.6963ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:45.747551863Z","hostname":"sock","pid":1574,"zone":"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\") failed: Property not found. retry in 979.367337ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:45.765067684Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\") failed: Property not found. retry in 952.310241ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:45.836116074Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139\") failed: Property not found. retry in 1.031566905s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:45.888112212Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\") failed: Property not found. retry in 531.622391ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:46.098559891Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\") failed: Property not found. retry in 1.098471489s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:46.22314702Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\") failed: Property not found. retry in 1.42509044s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:46.236177238Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\") failed: Property not found. retry in 898.744731ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:46.245409117Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\") failed: Property not found. retry in 1.109684958s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:46.255186179Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\") failed: Property not found. retry in 816.394968ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:46.266106741Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d\") failed: Property not found. retry in 833.460767ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:46.384653985Z","hostname":"sock","pid":1574,"zone":"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\") failed: Property not found. retry in 1.354438933s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:46.395733988Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\") failed: Property not found. retry in 653.21363ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:46.462199713Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2\") failed: Property not found. retry in 1.389498464s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:46.476146097Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\") failed: Property not found. retry in 966.495661ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:46.495485782Z","hostname":"sock","pid":1574,"zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\") failed: Property not found. retry in 984.509665ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:46.557249982Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\") failed: Property not found. retry in 936.053699ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:46.640450613Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3\") failed: Property not found. retry in 936.0703ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:46.650426397Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\") failed: Property not found. retry in 761.900712ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:46.747011847Z","hostname":"sock","pid":1574,"zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\") failed: Property not found. retry in 1.263432813s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:46.757075722Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\") failed: Property not found. retry in 1.29029559s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:46.782466891Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\") failed: Property not found. retry in 1.380278044s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:46.798118618Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139\") failed: Property not found. retry in 530.517883ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:46.933144035Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\") failed: Property not found. retry in 1.288943673s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:47.096590884Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\") failed: Property not found. retry in 583.778413ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:47.131211672Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\") failed: Property not found. retry in 1.292825821s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:47.155573154Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d\") failed: Property not found. retry in 1.011067084s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:47.233692657Z","hostname":"sock","pid":1574,"zone":"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\") failed: Property not found. retry in 1.09673981s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:47.338384358Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\") failed: Property not found. retry in 1.416434607s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:47.379993538Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\") failed: Property not found. retry in 1.268415135s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:47.48355943Z","hostname":"sock","pid":1574,"zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139\") failed: Property not found. retry in 932.882247ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:47.500837259Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\") failed: Property not found. retry in 936.49401ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:47.523660859Z","hostname":"sock","pid":1574,"zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\") failed: Property not found. retry in 910.062919ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:47.553335791Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\") failed: Property not found. retry in 641.946606ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:47.594406487Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3\") failed: Property not found. retry in 1.393042805s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:47.610974031Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\") failed: Property not found. retry in 1.361424824s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:47.679564682Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\") failed: Property not found. retry in 598.667356ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:47.732605558Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\") failed: Property not found. retry in 1.372863561s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:47.770155098Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2\") failed: Property not found. retry in 1.310040997s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:47.882365235Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\") failed: Property not found. retry in 739.454467ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:48.03725237Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\") failed: Property not found. retry in 1.427534772s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:48.086510577Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\") failed: Property not found. retry in 1.291220476s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:48.194865975Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\") failed: Property not found. retry in 1.253905581s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:48.251543978Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d\") failed: Property not found. retry in 1.297689263s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:48.264318243Z","hostname":"sock","pid":1574,"zone":"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\") failed: Property not found. retry in 534.51814ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:48.350321675Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\") failed: Property not found. retry in 560.360643ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:48.404097876Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139\") failed: Property not found. retry in 1.17754918s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:48.45027772Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\") failed: Property not found. retry in 783.009877ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:48.464342825Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\") failed: Property not found. retry in 1.367780868s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:48.478800223Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\") failed: Property not found. retry in 953.377158ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:48.491672589Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\") failed: Property not found. retry in 1.345810368s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:48.505502532Z","hostname":"sock","pid":1574,"zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\") failed: Property not found. retry in 813.385749ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:48.784936136Z","hostname":"sock","pid":1574,"zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\") failed: Property not found. retry in 1.290216205s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:48.802247885Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\") failed: Property not found. retry in 913.880671ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:48.816591062Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:48.856775281Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:48.857239185Z","hostname":"sock","pid":1574,"zone":"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\") failed: Property not found. retry in 905.412817ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:48.896097805Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\") failed: Property not found. retry in 1.499691543s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:48.975703028Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3\") failed: Property not found. retry in 1.353838495s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:49.017694822Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\") failed: Property not found. retry in 707.361955ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:49.056632962Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121\") failed: Property not found. retry in 67.597411ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:49.102626345Z","hostname":"sock","pid":1574,"zone":"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\") failed: Property not found. retry in 545.727787ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:49.157409554Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121\") failed: Property not found. retry in 91.537721ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:49.180168533Z","hostname":"sock","pid":1574,"zone":"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2\") failed: Property not found. retry in 1.268219169s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:49.204592466Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\") failed: Property not found. retry in 780.979284ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:49.268493432Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121\") failed: Property not found. retry in 142.100933ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:49.290733578Z","hostname":"sock","pid":1574,"zone":"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121\") failed: Property not found. retry in 480.779258ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:49.455094154Z","hostname":"sock","pid":1574,"zone":"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\") failed: Property not found. retry in 1.498900581s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:49.47599895Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\") failed: Property not found. retry in 691.342918ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:49.505113437Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\") failed: Property not found. retry in 850.079249ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:49.522692338Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\") failed: Property not found. retry in 971.508071ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:49.542660507Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d\") failed: Property not found. retry in 627.669981ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:49.580153397Z","hostname":"sock","pid":1574,"zone":"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\") failed: Property not found. retry in 1.073533277s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:49.628349866Z","hostname":"sock","pid":1574,"zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139\") failed: Property not found. retry in 947.922028ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:49.644561017Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\") failed: Property not found. retry in 865.63726ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:49.7161122Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\") failed: Property not found. retry in 640.58936ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:49.744148019Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\") failed: Property not found. retry in 708.141712ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:49.776938924Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\") failed: Property not found. retry in 905.566661ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:49.814678035Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\") failed: Property not found. retry in 741.321039ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:49.859374499Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\") failed: Property not found. retry in 1.348382996s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:49.870705883Z","hostname":"sock","pid":1574,"zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121\") failed: Property not found. retry in 776.060723ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:49.947820168Z","hostname":"sock","pid":1574,"zone":"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\") failed: Property not found. retry in 1.059487121s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:50.062763516Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\") failed: Property not found. retry in 989.208109ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:50.106853795Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\") failed: Property not found. retry in 1.094698433s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:50.211573396Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d\") failed: Property not found. retry in 1.126337399s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:50.221303438Z","hostname":"sock","pid":1574,"zone":"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3\") failed: Property not found. retry in 1.015291215s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:50.384181323Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\") failed: Property not found. retry in 808.144251ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:50.396918528Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\") failed: Property not found. retry in 1.153291797s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:50.407589338Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2\") failed: Property not found. retry in 791.462837ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:50.488863974Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\") failed: Property not found. retry in 1.027635018s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:50.500958354Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\") failed: Property not found. retry in 852.861202ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:50.511047629Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\") failed: Property not found. retry in 1.01856674s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:50.526296123Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\") failed: Property not found. retry in 1.447654688s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:50.594145699Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139\") failed: Property not found. retry in 661.941569ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:50.606192259Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\") failed: Property not found. retry in 1.258687046s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:50.621736365Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\") failed: Property not found. retry in 869.153455ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:50.716978685Z","hostname":"sock","pid":1574,"zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\") failed: Property not found. retry in 1.447959164s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:50.735400722Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121\") failed: Property not found. retry in 822.882254ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:50.749191985Z","hostname":"sock","pid":1574,"zone":"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\") failed: Property not found. retry in 1.132330404s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:50.990221923Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\") failed: Property not found. retry in 753.328235ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:51.110388559Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\") failed: Property not found. retry in 738.058133ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:51.138508719Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\") failed: Property not found. retry in 818.573515ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:51.221986831Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\") failed: Property not found. retry in 1.298774223s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:51.235775964Z","hostname":"sock","pid":1574,"zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139\") failed: Property not found. retry in 1.354780661s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:51.281740217Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2\") failed: Property not found. retry in 759.120247ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:51.296007203Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\") failed: Property not found. retry in 1.127010969s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:51.322170054Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d\") failed: Property not found. retry in 1.015340881s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:51.366962622Z","hostname":"sock","pid":1574,"zone":"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\") failed: Property not found. retry in 748.070027ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:51.382252127Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3\") failed: Property not found. retry in 1.415524044s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:51.414117494Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\") failed: Property not found. retry in 1.459840851s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:51.542504212Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\") failed: Property not found. retry in 598.272668ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:51.557905057Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\") failed: Property not found. retry in 1.062942911s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:51.574808823Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121\") failed: Property not found. retry in 519.503614ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:51.587219855Z","hostname":"sock","pid":1574,"zone":"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\") failed: Property not found. retry in 1.013232282s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:51.600347963Z","hostname":"sock","pid":1574,"zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\") failed: Property not found. retry in 1.09281792s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:51.882588968Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\") failed: Property not found. retry in 1.494572485s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:51.895930037Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\") failed: Property not found. retry in 518.339973ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:51.909477018Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\") failed: Property not found. retry in 1.000319167s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:52.058826342Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2\") failed: Property not found. retry in 914.191775ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:52.071862509Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\") failed: Property not found. retry in 1.148223142s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:52.084759486Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121\") failed: Property not found. retry in 614.860385ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:52.124647083Z","hostname":"sock","pid":1574,"zone":"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\") failed: Property not found. retry in 1.299012844s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:52.146932099Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\") failed: Property not found. retry in 940.070496ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:52.163813985Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\") failed: Property not found. retry in 692.850737ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:52.181168974Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\") failed: Property not found. retry in 1.387421111s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:52.197461076Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d\") failed: Property not found. retry in 1.147582365s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:52.405408827Z","hostname":"sock","pid":1574,"zone":"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\") failed: Property not found. retry in 1.084788s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:52.44608779Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\") failed: Property not found. retry in 1.240935491s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:52.464974911Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\") failed: Property not found. retry in 562.352171ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:52.553069398Z","hostname":"sock","pid":1574,"zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\") failed: Property not found. retry in 901.832666ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:52.634625576Z","hostname":"sock","pid":1574,"zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139\") failed: Property not found. retry in 1.374259698s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:52.655007708Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\") failed: Property not found. retry in 947.919059ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:52.673735168Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121\") failed: Property not found. retry in 1.003921101s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:52.76113458Z","hostname":"sock","pid":1574,"zone":"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3\") failed: Property not found. retry in 655.666369ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:52.844668383Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\") failed: Property not found. retry in 946.205415ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:52.88725486Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\") failed: Property not found. retry in 1.321030255s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:52.987130295Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2\") failed: Property not found. retry in 1.23198961s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:52.999396417Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\") failed: Property not found. retry in 1.333693259s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:53.015371636Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\") failed: Property not found. retry in 743.959499ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:53.070207625Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\") failed: Property not found. retry in 1.466175917s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:53.115855265Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\") failed: Property not found. retry in 891.49857ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:53.129599678Z","hostname":"sock","pid":1574,"zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\") failed: Property not found. retry in 934.370137ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:53.246733961Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\") failed: Property not found. retry in 1.335710552s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:53.403351359Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\") failed: Property not found. retry in 1.112781494s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:53.458224958Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3\") failed: Property not found. retry in 787.167368ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:53.511931009Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d\") failed: Property not found. retry in 1.403115376s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:53.543375759Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740\") failed: Property not found. retry in 901.993241ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:53.555217952Z","hostname":"sock","pid":1574,"zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d\") failed: Property not found. retry in 1.273127232s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:53.5657274Z","hostname":"sock","pid":1574,"zone":"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea\") failed: Property not found. retry in 1.460781255s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:53.59527312Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\") failed: Property not found. retry in 634.763555ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:53.639894853Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\") failed: Property not found. retry in 1.026065272s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:53.719470021Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121\") failed: Property not found. retry in 1.025471691s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:53.779961697Z","hostname":"sock","pid":1574,"zone":"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13\") failed: Property not found. retry in 1.288953416s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:53.825839916Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\") failed: Property not found. retry in 921.748132ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:53.844368818Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77\") failed: Property not found. retry in 1.0454798s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:54.033145096Z","hostname":"sock","pid":1574,"zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139\") failed: Property not found. retry in 1.17301794s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:54.043355792Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\") failed: Property not found. retry in 695.750088ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:54.191724618Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2\") failed: Property not found. retry in 1.05625885s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:54.248524781Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd\") failed: Property not found. retry in 1.329219021s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:54.288170938Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3\") failed: Property not found. retry in 1.159819367s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:54.311055978Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563\") failed: Property not found. retry in 890.189091ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:54.320767121Z","hostname":"sock","pid":1574,"zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4\") failed: Property not found. retry in 819.025643ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:54.357889787Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4\") failed: Property not found. retry in 534.063993ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:54.581820507Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095\") failed: Property not found. retry in 1.45415573s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:54.592461027Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854\") failed: Property not found. retry in 878.605805ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:54.756132517Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a\") failed: Property not found. retry in 1.015361776s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:54.76863714Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730\") failed: Property not found. retry in 1.495848721s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:54.779669912Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121\") failed: Property not found. retry in 1.288650686s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:54.821645045Z","hostname":"sock","pid":1574,"zone":"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb\") failed: Property not found. retry in 667.944112ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:54.897473181Z","hostname":"sock","pid":1574,"zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121\") failed: Property not found. retry in 847.048605ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:56.130434496Z","hostname":"sock","pid":1574,"zone":"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121\") failed: Property not found. retry in 1.477135102s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:56.999548987Z","hostname":"sock","pid":1574,"zone":"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:00:58.42137765Z","hostname":"sock","pid":1574,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:420","component":"producer-server","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/producer/src/lib.rs:420","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers)\"","delay":"14.738907646s"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:58.517269355Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3317","zone":"oxz_nexus_16a7178a-dbb6-4392-b395-7bddb3817d13"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:58.517343585Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3317","zone":"oxz_clickhouse_e5181def-6535-4350-b7f4-acf3785cf121"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:58.517351515Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3317","zone":"oxz_crucible_6cdeb736-9e69-46f9-afb0-59255807dbe4"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:58.517360845Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3317","zone":"oxz_crucible_pantry_154903a9-f4ab-4352-b947-96c5fce154b3"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:58.517367116Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3317","zone":"oxz_oximeter_84b5a094-9eb0-4ff8-be58-5c4913059740"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:58.517372925Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3317","zone":"oxz_nexus_9b2d2174-b236-4546-bace-300e05afe3c2"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:58.517389826Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3317","zone":"oxz_nexus_3dce42ed-74cb-4c8b-9326-8273a1efd563"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:58.517396416Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3317","zone":"oxz_crucible_9b5ceaf3-96d0-4f36-882b-965f9acc31c4"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:58.517402926Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3317","zone":"oxz_crucible_c7378503-c5b8-4eee-91c0-2c15e3708aea"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:58.517409596Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3317","zone":"oxz_crucible_abc9c059-37e5-408c-b8e0-9fdb2f94fcfb"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:58.517416796Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3317","zone":"oxz_crucible_pantry_1b77d62e-b462-4293-950d-00a565b722bd"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:58.517423576Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3317","zone":"oxz_external_dns_be35e232-dd9b-4d59-b4e1-eadb7f69bb77"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:58.517433066Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3317","zone":"oxz_crucible_08aaade2-f75a-444c-b97a-b4d3bf97b095"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:58.517441946Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3317","zone":"oxz_external_dns_14e637a2-51ce-4cac-b831-85b757e5c55d"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:58.517454806Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3317","zone":"oxz_crucible_ced4eec8-4bf4-46b3-843c-91242defe60a"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:58.517461506Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3317","zone":"oxz_crucible_04a81970-7129-40c7-b6f5-36219a8ad854"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:58.517468146Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3317","zone":"oxz_crucible_pantry_40c1b925-76fc-473c-9317-ffcfbf8aef2d"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:58.517475276Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3317","zone":"oxz_crucible_948190c1-2519-4134-a120-8d6900d50730"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:58.517481616Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:3317","zone":"oxz_crucible_090b4de4-d9d1-47aa-96a9-4cc472900139"} {"msg":"Writing ledger to /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:58.518067151Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:200"} {"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:58.519600522Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:133","path":"\"/pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-zones.json\""} {"msg":"Writing ledger to /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:58.519640293Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:200"} {"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:58.526971157Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:133","path":"\"/pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-zones.json\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:58.530617924Z","hostname":"sock","pid":1574,"uri":"/omicron-zones","method":"PUT","req_id":"ce5e416f-1ea1-44c4-8d65-b8a45378a6d9","remote_addr":"[fd00:1122:3344:101::1]:49408","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":40760412,"response_code":"204"} {"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:58.543879653Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1448"} {"msg":"Writing ledger to /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:58.543904893Z","hostname":"sock","pid":1574,"component":"RSS","file":"common/src/ledger.rs:200"} {"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:58.544086055Z","hostname":"sock","pid":1574,"component":"RSS","file":"common/src/ledger.rs:133","path":"\"/pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/rss-plan-completed.marker\""} {"msg":"Writing ledger to /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:58.544095135Z","hostname":"sock","pid":1574,"component":"RSS","file":"common/src/ledger.rs:200"} {"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:58.544204776Z","hostname":"sock","pid":1574,"component":"RSS","file":"common/src/ledger.rs:133","path":"\"/pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/rss-plan-completed.marker\""} {"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:58.697112996Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:787"} {"msg":"Nexus address: [fd00:1122:3344:101::a]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:58.709144445Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:799"} {"msg":"rack_network_config: RackNetworkConfigV2 {\n bfd: [],\n bgp: [],\n infra_ip_first: 10.151.2.174,\n infra_ip_last: 10.151.2.174,\n ports: [\n PortConfigV2 {\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.2.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n autoneg: false,\n bgp_peers: [],\n lldp: None,\n port: \"qsfp0\",\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.2.100,\n rib_priority: None,\n vlan_id: None,\n },\n ],\n switch: Switch0,\n tx_eq: None,\n uplink_port_fec: Some(\n None,\n ),\n uplink_port_speed: Speed40G,\n },\n ],\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:58.771669262Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1013"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/f35dae23-a43a-4d77-a204-fab3c4bdfff3/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:58.974332173Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1075"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/f35dae23-a43a-4d77-a204-fab3c4bdfff3/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:59.099833466Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1075"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/f35dae23-a43a-4d77-a204-fab3c4bdfff3/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:59.231695252Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1075"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/f35dae23-a43a-4d77-a204-fab3c4bdfff3/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:59.423789725Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1075"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/f35dae23-a43a-4d77-a204-fab3c4bdfff3/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:59.520189614Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1075"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/f35dae23-a43a-4d77-a204-fab3c4bdfff3/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:59.693649118Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1075"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/f35dae23-a43a-4d77-a204-fab3c4bdfff3/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:59.867589345Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1075"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:59.926566105Z","hostname":"sock","pid":1574,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:62500"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:00:59.926906207Z","hostname":"sock","pid":1574,"uri":"/baseboard","method":"GET","req_id":"4a375aa0-96bc-4b42-8609-1d4905123960","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:62500","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":34,"response_code":"200"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/f35dae23-a43a-4d77-a204-fab3c4bdfff3/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:01:00.215056546Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1075"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/f35dae23-a43a-4d77-a204-fab3c4bdfff3/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:01:00.547542256Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1075"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/f35dae23-a43a-4d77-a204-fab3c4bdfff3/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:01:01.051129111Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1075"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/f35dae23-a43a-4d77-a204-fab3c4bdfff3/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:01:01.702934621Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1075"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/f35dae23-a43a-4d77-a204-fab3c4bdfff3/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:01:02.268328788Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1075"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/f35dae23-a43a-4d77-a204-fab3c4bdfff3/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:01:02.646856474Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1075"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/f35dae23-a43a-4d77-a204-fab3c4bdfff3/initialization-complete)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:01:03.405746679Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1075"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"84bce97e-d9b8-4a7d-8685-ea1557aa601a\", \"content-length\": \"173\", \"date\": \"Mon, 18 Nov 2024 21:01:04 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: rack with id \\\"f35dae23-a43a-4d77-a204-fab3c4bdfff3\\\"\", request_id: \"84bce97e-d9b8-4a7d-8685-ea1557aa601a\" }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:01:04.427168036Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1075"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"46e47d31-ece6-4c88-b51c-9b6087ff22da\", \"content-length\": \"173\", \"date\": \"Mon, 18 Nov 2024 21:01:05 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: rack with id \\\"f35dae23-a43a-4d77-a204-fab3c4bdfff3\\\"\", request_id: \"46e47d31-ece6-4c88-b51c-9b6087ff22da\" }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:01:05.196733586Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1075"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7eb28efc-69e5-423d-8e9f-77fcfa52f3c8\", \"content-length\": \"173\", \"date\": \"Mon, 18 Nov 2024 21:01:06 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"79cbda5a-2e27-489e-945d-860d86dec819\\\"\", request_id: \"7eb28efc-69e5-423d-8e9f-77fcfa52f3c8\" }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:01:07.703647612Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1075"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4fba9681-0532-4680-8e71-0f0015298651\", \"content-length\": \"173\", \"date\": \"Mon, 18 Nov 2024 21:01:10 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"79cbda5a-2e27-489e-945d-860d86dec819\\\"\", request_id: \"4fba9681-0532-4680-8e71-0f0015298651\" }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:01:10.079318639Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1075"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"937df14c-c8f0-42fe-abcd-8bc441f61f75\", \"content-length\": \"173\", \"date\": \"Mon, 18 Nov 2024 21:01:12 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"79cbda5a-2e27-489e-945d-860d86dec819\\\"\", request_id: \"937df14c-c8f0-42fe-abcd-8bc441f61f75\" }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:01:12.923670292Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1075"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"49f6c7db-3163-458e-a06a-feb5a426cd9e\", \"content-length\": \"173\", \"date\": \"Mon, 18 Nov 2024 21:01:16 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"79cbda5a-2e27-489e-945d-860d86dec819\\\"\", request_id: \"49f6c7db-3163-458e-a06a-feb5a426cd9e\" }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:01:16.140963146Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1075"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b41a8bfd-646a-49cd-aff3-20d915018414\", \"content-length\": \"173\", \"date\": \"Mon, 18 Nov 2024 21:01:19 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"79cbda5a-2e27-489e-945d-860d86dec819\\\"\", request_id: \"b41a8bfd-646a-49cd-aff3-20d915018414\" }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:01:19.960681973Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1075"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a12ad79d-1147-4ad4-b8b8-add904c7217c\", \"content-length\": \"173\", \"date\": \"Mon, 18 Nov 2024 21:01:23 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"79cbda5a-2e27-489e-945d-860d86dec819\\\"\", request_id: \"a12ad79d-1147-4ad4-b8b8-add904c7217c\" }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:01:23.537193026Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1075"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:01:26.055776689Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:58808","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","component":"producer-server","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::d]:50855"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:01:26.097397349Z","hostname":"sock","pid":1574,"uri":"/79cbda5a-2e27-489e-945d-860d86dec819","method":"GET","req_id":"8860ec09-024c-4af7-9c09-e5be03d55332","remote_addr":"[fd00:1122:3344:101::d]:50855","local_addr":"[fd00:1122:3344:101::1]:58808","component":"dropshot","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","component":"producer-server","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":40088,"response_code":"200"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"28e9784d-c49e-4bb1-a52c-d201f3f29116\", \"content-length\": \"173\", \"date\": \"Mon, 18 Nov 2024 21:01:28 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"79cbda5a-2e27-489e-945d-860d86dec819\\\"\", request_id: \"28e9784d-c49e-4bb1-a52c-d201f3f29116\" }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:01:28.987476683Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1075"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:01:29.966072032Z","hostname":"sock","pid":1574,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:57040"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:01:29.966141712Z","hostname":"sock","pid":1574,"uri":"/baseboard","method":"GET","req_id":"af828ada-07dd-4938-a3ab-a4946ab5c6f7","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:57040","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":25,"response_code":"200"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"402f4da6-3667-41b3-84f2-2c2336af8433\", \"content-length\": \"173\", \"date\": \"Mon, 18 Nov 2024 21:01:34 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"79cbda5a-2e27-489e-945d-860d86dec819\\\"\", request_id: \"402f4da6-3667-41b3-84f2-2c2336af8433\" }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:01:34.698779777Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1075"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"222eaed8-a2b4-4133-8aa4-1cebaba03892\", \"content-length\": \"173\", \"date\": \"Mon, 18 Nov 2024 21:01:39 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"79cbda5a-2e27-489e-945d-860d86dec819\\\"\", request_id: \"222eaed8-a2b4-4133-8aa4-1cebaba03892\" }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:01:39.876473032Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1075"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"665e42cd-1d4f-4b52-9771-3a86c9cf12f4\", \"content-length\": \"173\", \"date\": \"Mon, 18 Nov 2024 21:01:48 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"79cbda5a-2e27-489e-945d-860d86dec819\\\"\", request_id: \"665e42cd-1d4f-4b52-9771-3a86c9cf12f4\" }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:01:49.208188167Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1075"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"089a4019-883a-416f-af7b-288a44c3f922\", \"content-length\": \"173\", \"date\": \"Mon, 18 Nov 2024 21:01:56 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"79cbda5a-2e27-489e-945d-860d86dec819\\\"\", request_id: \"089a4019-883a-416f-af7b-288a44c3f922\" }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:01:57.047158351Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1075"} {"msg":"Successfully put SledAgentInfo to nexus","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:01:57.231440005Z","hostname":"sock","pid":1574,"component":"NexusNotifierTask","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/nexus.rs:500","new_generation":"1","old_generation":"1"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:01:57.326112072Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:45499"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:01:57.327405061Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:01:57.327412831Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:01:57.328452949Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:01:57.330281353Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:01:57.342947697Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:01:57.343000758Z","hostname":"sock","pid":1574,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_473f0b4e-a714-496d-8d5d-665782986893"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:01:57.368447817Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:01:57.368500658Z","hostname":"sock","pid":1574,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:01:57.389813357Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:01:57.389867207Z","hostname":"sock","pid":1574,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:01:57.414104118Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:01:57.414152358Z","hostname":"sock","pid":1574,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_2536a198-fc1d-4175-aab5-a62f1506f3e4"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:01:57.434658001Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:01:57.434707352Z","hostname":"sock","pid":1574,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_dd78d2d6-0d80-45fe-9f73-e75ef7da86ee"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:01:57.454501989Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:01:57.45456959Z","hostname":"sock","pid":1574,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_28b058a3-a4b5-473e-8580-f5c606b6d30b"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:01:57.475820168Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:01:57.475886879Z","hostname":"sock","pid":1574,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_80f7cc50-8d94-4fb0-b296-446e3acf3366"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:01:57.495661866Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:01:57.495716277Z","hostname":"sock","pid":1574,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_feac0a3f-e4dc-4a16-9eeb-839f74ad9665"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:01:57.514651598Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:01:57.514705508Z","hostname":"sock","pid":1574,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_f334d74e-134c-4206-a5ba-9a6ac6ea7954"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:01:57.526090693Z","hostname":"sock","pid":1574,"uri":"/inventory","method":"GET","req_id":"c8dda2d7-fd13-4db9-9bf0-939a9cce47ae","remote_addr":"[fd00:1122:3344:101::c]:45499","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":199753,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:00.005523624Z","hostname":"sock","pid":1574,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:62288"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:00.005626465Z","hostname":"sock","pid":1574,"uri":"/baseboard","method":"GET","req_id":"d08e05a3-2d44-4d6c-a196-b3f2bbf7a9bd","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:62288","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":24,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:05.657724306Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:33577"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:05.658696895Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"b8fa9819-5632-49b0-b137-5963f4a4df71","remote_addr":"[fd00:1122:3344:101::a]:33577","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":850,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:05.746300279Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:05.74642688Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"b741baa0-1732-4769-b8ce-682da95a94f3","remote_addr":"[fd00:1122:3344:101::a]:33577","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":634,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:05.817727011Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:44712"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:05.818092074Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"ff4e9c0b-a4ee-46c3-a76d-a320f9de1c59","remote_addr":"[fd00:1122:3344:101::a]:44712","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":289,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:05.919676872Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"24084a46-f08b-4a7c-9ef3-ec76f843012d","remote_addr":"[fd00:1122:3344:101::a]:33577","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":2707,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:06.191115746Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:51259"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:06.191206887Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"7fb9ff9e-c396-4996-8864-093a2c5c0140","remote_addr":"[fd00:1122:3344:101::b]:51259","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":40,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:06.234228768Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:06.234259248Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"7b88865e-bdc0-41d5-a88f-52d52204f912","remote_addr":"[fd00:1122:3344:101::b]:51259","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":53,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:06.24396412Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"facb92e8-8943-4592-8a1c-44ff4e123d8d","remote_addr":"[fd00:1122:3344:101::b]:51259","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":163,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:06.281563971Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:34104"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:06.281774622Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"14a5818f-3ddd-451a-98e9-5ecd1ca58976","remote_addr":"[fd00:1122:3344:101::b]:34104","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":161,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:06.397941659Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:41974"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:06.39807038Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"03c97e9e-3769-4ad5-a23c-04d898b167fa","remote_addr":"[fd00:1122:3344:101::c]:41974","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":110,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:06.421448454Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:49242"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:06.421530335Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"e5aed780-f0dc-4c4a-a76f-e6eefebb0dd1","remote_addr":"[fd00:1122:3344:101::c]:49242","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":23,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:06.479631288Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:06.479695698Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"cc7afdb5-ca45-4047-bd69-955ce063e5aa","remote_addr":"[fd00:1122:3344:101::c]:49242","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":104,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:06.490050466Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"e1c0b295-11d5-444f-ba7a-6a4ae44e6921","remote_addr":"[fd00:1122:3344:101::c]:49242","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":178,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:13.746394823Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:57130"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:13.748392827Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3b3dea8a-e84f-4acf-bafc-3b0e4a815dc5, kind: Service { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 }, name: Name(\"external-dns-be35e232-dd9b-4d59-b4e1-eadb7f69bb77\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 109])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 95dce781-7d6b-4bf1-88f1-880b60d0adba, kind: Service { id: 14e637a2-51ce-4cac-b831-85b757e5c55d }, name: Name(\"external-dns-14e637a2-51ce-4cac-b831-85b757e5c55d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 238])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 5dbd0a76-35d9-4b29-8800-824968d1caba, kind: Service { id: 9b2d2174-b236-4546-bace-300e05afe3c2 }, name: Name(\"nexus-9b2d2174-b236-4546-bace-300e05afe3c2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 121])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: db0c0ea5-0d8f-4f9d-9979-f3989ce962ea, kind: Service { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 }, name: Name(\"nexus-16a7178a-dbb6-4392-b395-7bddb3817d13\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e67bea26-1a67-4166-875f-3cab431644e6, kind: Service { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 }, name: Name(\"nexus-3dce42ed-74cb-4c8b-9326-8273a1efd563\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 239])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:13.748563359Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:13.748598289Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:13.748606629Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:13.748612869Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:13.748622379Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:13.748637089Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:13.748667779Z","hostname":"sock","pid":1574,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"187c65a1-8acf-403d-ba12-872134437775","remote_addr":"[fd00:1122:3344:101::a]:57130","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":2166,"response_code":"204"} {"msg":"Handoff to Nexus is complete","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:13.750306822Z","hostname":"sock","pid":1574,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1085"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:14.027573969Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:47240"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:14.028327125Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:14.028341675Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:14.028397106Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:14.028680408Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:14.041156391Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:14.041247342Z","hostname":"sock","pid":1574,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_473f0b4e-a714-496d-8d5d-665782986893"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:14.065322741Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:14.065372891Z","hostname":"sock","pid":1574,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:14.087050293Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:14.087103553Z","hostname":"sock","pid":1574,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:14.108236971Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:14.108294222Z","hostname":"sock","pid":1574,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_2536a198-fc1d-4175-aab5-a62f1506f3e4"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:14.128284421Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:14.128340271Z","hostname":"sock","pid":1574,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_dd78d2d6-0d80-45fe-9f73-e75ef7da86ee"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:14.149323037Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:14.149369808Z","hostname":"sock","pid":1574,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_28b058a3-a4b5-473e-8580-f5c606b6d30b"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:14.16974652Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:14.16981071Z","hostname":"sock","pid":1574,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_80f7cc50-8d94-4fb0-b296-446e3acf3366"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:14.190328773Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:14.190389444Z","hostname":"sock","pid":1574,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_feac0a3f-e4dc-4a16-9eeb-839f74ad9665"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:14.210566704Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:14.210620625Z","hostname":"sock","pid":1574,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_f334d74e-134c-4206-a5ba-9a6ac6ea7954"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:14.222440143Z","hostname":"sock","pid":1574,"uri":"/inventory","method":"GET","req_id":"5163f9de-8e3b-4174-a2d1-2435c8656a00","remote_addr":"[fd00:1122:3344:101::a]:47240","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":194141,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:14.377115527Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:52284"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:14.377378938Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3b3dea8a-e84f-4acf-bafc-3b0e4a815dc5, kind: Service { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 }, name: Name(\"external-dns-be35e232-dd9b-4d59-b4e1-eadb7f69bb77\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 109])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 95dce781-7d6b-4bf1-88f1-880b60d0adba, kind: Service { id: 14e637a2-51ce-4cac-b831-85b757e5c55d }, name: Name(\"external-dns-14e637a2-51ce-4cac-b831-85b757e5c55d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 238])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 5dbd0a76-35d9-4b29-8800-824968d1caba, kind: Service { id: 9b2d2174-b236-4546-bace-300e05afe3c2 }, name: Name(\"nexus-9b2d2174-b236-4546-bace-300e05afe3c2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 121])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: db0c0ea5-0d8f-4f9d-9979-f3989ce962ea, kind: Service { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 }, name: Name(\"nexus-16a7178a-dbb6-4392-b395-7bddb3817d13\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e67bea26-1a67-4166-875f-3cab431644e6, kind: Service { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 }, name: Name(\"nexus-3dce42ed-74cb-4c8b-9326-8273a1efd563\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 239])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:14.377436899Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:14.377478139Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:14.377489329Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:14.377502239Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:14.37753056Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:14.37754653Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:14.37756985Z","hostname":"sock","pid":1574,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"0f20b293-4477-4838-aa95-ac7a4570dbcd","remote_addr":"[fd00:1122:3344:101::a]:52284","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":372,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:14.466825585Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:55892"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:14.467028107Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3b3dea8a-e84f-4acf-bafc-3b0e4a815dc5, kind: Service { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 }, name: Name(\"external-dns-be35e232-dd9b-4d59-b4e1-eadb7f69bb77\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 109])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 95dce781-7d6b-4bf1-88f1-880b60d0adba, kind: Service { id: 14e637a2-51ce-4cac-b831-85b757e5c55d }, name: Name(\"external-dns-14e637a2-51ce-4cac-b831-85b757e5c55d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 238])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 5dbd0a76-35d9-4b29-8800-824968d1caba, kind: Service { id: 9b2d2174-b236-4546-bace-300e05afe3c2 }, name: Name(\"nexus-9b2d2174-b236-4546-bace-300e05afe3c2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 121])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: db0c0ea5-0d8f-4f9d-9979-f3989ce962ea, kind: Service { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 }, name: Name(\"nexus-16a7178a-dbb6-4392-b395-7bddb3817d13\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e67bea26-1a67-4166-875f-3cab431644e6, kind: Service { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 }, name: Name(\"nexus-3dce42ed-74cb-4c8b-9326-8273a1efd563\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 239])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:14.467093138Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:14.467121808Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:14.467142038Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:14.467161708Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:14.467183978Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:14.467206939Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:14.467227939Z","hostname":"sock","pid":1574,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"adc1d8d4-4115-4226-b5ea-a93739d682bf","remote_addr":"[fd00:1122:3344:101::c]:55892","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":330,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:15.012107552Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:62480"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:15.012330354Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3b3dea8a-e84f-4acf-bafc-3b0e4a815dc5, kind: Service { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 }, name: Name(\"external-dns-be35e232-dd9b-4d59-b4e1-eadb7f69bb77\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 109])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 95dce781-7d6b-4bf1-88f1-880b60d0adba, kind: Service { id: 14e637a2-51ce-4cac-b831-85b757e5c55d }, name: Name(\"external-dns-14e637a2-51ce-4cac-b831-85b757e5c55d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 238])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 5dbd0a76-35d9-4b29-8800-824968d1caba, kind: Service { id: 9b2d2174-b236-4546-bace-300e05afe3c2 }, name: Name(\"nexus-9b2d2174-b236-4546-bace-300e05afe3c2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 121])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: db0c0ea5-0d8f-4f9d-9979-f3989ce962ea, kind: Service { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 }, name: Name(\"nexus-16a7178a-dbb6-4392-b395-7bddb3817d13\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e67bea26-1a67-4166-875f-3cab431644e6, kind: Service { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 }, name: Name(\"nexus-3dce42ed-74cb-4c8b-9326-8273a1efd563\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 239])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:15.012381174Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:15.012415224Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:15.012445855Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:15.012453665Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:15.012460955Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:15.012476545Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:15.012495825Z","hostname":"sock","pid":1574,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"35fb4749-bfa1-4c51-a730-46c135e9a7ee","remote_addr":"[fd00:1122:3344:101::b]:62480","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":304,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:18.872403492Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:55960"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:18.872572553Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"4dceb644-18c5-4355-aeb9-db458b79cd04","remote_addr":"[fd00:1122:3344:101::a]:55960","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":85,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:18.917925721Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:18.917966072Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"7dea4902-6b06-40f7-aabd-7203595f324b","remote_addr":"[fd00:1122:3344:101::a]:55960","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":91,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:18.928071497Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"bd035290-ee8a-4c87-a416-f89396b2d8b8","remote_addr":"[fd00:1122:3344:101::a]:55960","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":198,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:30.038970031Z","hostname":"sock","pid":1574,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:64186"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:30.039073082Z","hostname":"sock","pid":1574,"uri":"/baseboard","method":"GET","req_id":"fe04191a-146f-48ac-9d1e-0a2633520a6a","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:64186","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":31,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:35.56847121Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:50228"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:35.568593091Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"6f4103dc-65f5-48ba-be7e-693d5a0c2f37","remote_addr":"[fd00:1122:3344:101::a]:50228","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":53,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:35.588889343Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:46587"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:35.589063174Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"8c8e0308-b22b-4a2c-91f0-af3ba1387b87","remote_addr":"[fd00:1122:3344:101::a]:46587","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":123,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:35.64358743Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:35.6436117Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"7243dac9-9bc8-4f85-9674-8e59e962967c","remote_addr":"[fd00:1122:3344:101::a]:50228","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":64,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:35.652028763Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"390d1a71-3929-4648-bce7-5c9f09e7ff1a","remote_addr":"[fd00:1122:3344:101::a]:50228","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":172,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:35.718582269Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:43699"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:35.719247174Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:35.791905696Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:35.855049637Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"465dd99a-bab7-4956-8450-2a286715a9e2","remote_addr":"[fd00:1122:3344:101::a]:43699","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":136410,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:35.855572581Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"7855d225-832c-4d89-8a6e-4aeb3d3787aa","remote_addr":"[fd00:1122:3344:101::a]:43699","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":245,"response_code":"200"} {"msg":"Attempting to update network config with generation: 2, current_generation: 1","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:35.890690443Z","hostname":"sock","pid":1574,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:35.890816114Z","hostname":"sock","pid":1574,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:200"} {"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:35.892407696Z","hostname":"sock","pid":1574,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:133","path":"\"/pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/cluster/bootstore-network-config.json\""} {"msg":"Writing ledger to /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:35.892417166Z","hostname":"sock","pid":1574,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:200"} {"msg":"Successfully wrote to ledger","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:35.893607684Z","hostname":"sock","pid":1574,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:133","path":"\"/pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/cluster/bootstore-network-config.json\""} {"msg":"Broadcasting network config with generation 2","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:35.893643885Z","hostname":"sock","pid":1574,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:35.893648815Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"PUT","req_id":"e9857a3c-5d39-46c9-9b83-4553cc942424","remote_addr":"[fd00:1122:3344:101::a]:43699","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":3573,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:36.279615974Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:60789"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:36.279744895Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"b02788a0-d076-4097-98ac-b134e9c6d182","remote_addr":"[fd00:1122:3344:101::b]:60789","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":59,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:36.281894871Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:58557"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:36.282076862Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"e48f23d6-32c5-47c6-84a6-38a51483ef66","remote_addr":"[fd00:1122:3344:101::b]:58557","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":120,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:36.296403899Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:36.296441269Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"9a1881a6-3015-4863-b3bd-329b22336a4a","remote_addr":"[fd00:1122:3344:101::b]:60789","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":71,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:36.307482701Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"3e240716-723c-4dad-8389-c0619642d14b","remote_addr":"[fd00:1122:3344:101::b]:60789","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":185,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:36.389456003Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:58682"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:36.389540783Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:36.428503064Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:36.480682393Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"adfda41f-9314-4129-8b1b-96f5fa53931d","remote_addr":"[fd00:1122:3344:101::b]:58682","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":91175,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:36.480890455Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:53093"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:36.481055186Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"e41b96f2-493f-45aa-b575-be53cb009acc","remote_addr":"[fd00:1122:3344:101::c]:53093","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":103,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:36.481258267Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"8f2f4805-5370-4ecf-8d83-e8fdff95b938","remote_addr":"[fd00:1122:3344:101::b]:58682","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":232,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:36.512765693Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:42130"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:36.513562528Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"734bb86b-48da-4a60-9b4d-821ee3bfcb0e","remote_addr":"[fd00:1122:3344:101::c]:42130","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":198,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:36.527890535Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:36.527917375Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"6d2181dd-54fb-4943-807c-6cd3eddabbee","remote_addr":"[fd00:1122:3344:101::c]:53093","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":108,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:36.538962328Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"60be7f1c-d8b9-4aec-a3c9-02059eeba98a","remote_addr":"[fd00:1122:3344:101::c]:53093","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":264,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:36.544384098Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:56605"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:36.544477569Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:36.580087455Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:36.627860201Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"e12ece70-0212-4a59-ac37-8335438bcd59","remote_addr":"[fd00:1122:3344:101::c]:56605","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":83419,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:02:36.628202973Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"9596c2c5-e53f-4c6f-806c-167d8d8fbd17","remote_addr":"[fd00:1122:3344:101::c]:56605","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":125,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:00.077875836Z","hostname":"sock","pid":1574,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:42690"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:00.077980797Z","hostname":"sock","pid":1574,"uri":"/baseboard","method":"GET","req_id":"2496b77c-f617-463f-9081-4f15270a7b3a","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:42690","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":33,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:05.706888476Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:61429"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:05.706994947Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"609ba85a-1b35-4dfd-857f-71ac201168c1","remote_addr":"[fd00:1122:3344:101::a]:61429","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":47,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:05.721991269Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:05.722026289Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"0f5f7847-777f-45ec-960f-43ef1a908984","remote_addr":"[fd00:1122:3344:101::a]:61429","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":58,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:05.859178312Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:51415"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:05.859362663Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"fa8d2332-29df-4470-aa1c-03f32c7359ae","remote_addr":"[fd00:1122:3344:101::a]:51415","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":135,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:05.86698524Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"b4f06fd0-90ba-4cb0-b5ca-2241a0fbd6c0","remote_addr":"[fd00:1122:3344:101::a]:61429","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":152,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:05.958399132Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:62692"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:05.958481072Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:05.993257752Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:06.042533839Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"215861c8-ed83-4d22-8762-b8559c7d39d8","remote_addr":"[fd00:1122:3344:101::a]:62692","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":84083,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:06.042847732Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"47460365-fde6-44bd-9078-3190a8b73022","remote_addr":"[fd00:1122:3344:101::a]:62692","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":109,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:06.469167192Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:41637"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:06.469345203Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"7a317d05-c74f-48b9-a47f-20387fef2f43","remote_addr":"[fd00:1122:3344:101::b]:41637","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":127,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:06.546368227Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:59958"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:06.546450508Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"a55a42d1-a82f-4e5a-a246-785153836448","remote_addr":"[fd00:1122:3344:101::b]:59958","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":47,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:06.561430409Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:06.56145281Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"b96369fe-a748-4c27-8445-f18de7a1e1f7","remote_addr":"[fd00:1122:3344:101::b]:59958","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":61,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:06.571353363Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"9fd326f9-4084-41ed-9058-be58eae72eef","remote_addr":"[fd00:1122:3344:101::b]:59958","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":220,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:06.595220061Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:40209"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:06.595281292Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"09052935-e9c0-41d7-a297-7cf5fb20140d","remote_addr":"[fd00:1122:3344:101::c]:40209","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":31,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:06.637243105Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:06.637271345Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"b732abbc-10b2-4dc6-8e3e-b2ddc5e852e5","remote_addr":"[fd00:1122:3344:101::c]:40209","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":57,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:06.668224586Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:60972"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:06.668411147Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"453f54e7-ba9a-4125-9e2b-e3b16dc3b678","remote_addr":"[fd00:1122:3344:101::c]:60972","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":134,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:06.671530251Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:44430"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:06.671618281Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:06.705301002Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:06.759058823Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"37b756a3-a3ba-4adc-9f94-bb9d3500d94b","remote_addr":"[fd00:1122:3344:101::b]:44430","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":87461,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:06.759453036Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:50327"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:06.759598047Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"21289a63-778a-472d-81be-eb726d6189f8","remote_addr":"[fd00:1122:3344:101::b]:44430","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":247,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:06.759695298Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"abc99a8b-b356-4645-a715-7c7fc660098f","remote_addr":"[fd00:1122:3344:101::c]:40209","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":358,"response_code":"204"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:06.759745279Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:06.7934907Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:06.847682344Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"856bbea0-f29a-4391-b115-5626f9f1bfd7","remote_addr":"[fd00:1122:3344:101::c]:50327","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":88065,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:06.848039057Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"05086804-659a-4b3d-8d6d-96d5a2689aa9","remote_addr":"[fd00:1122:3344:101::c]:50327","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":131,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:17.873611414Z","hostname":"sock","pid":1574,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:30.111259852Z","hostname":"sock","pid":1574,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:60808"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:30.111364753Z","hostname":"sock","pid":1574,"uri":"/baseboard","method":"GET","req_id":"34323951-d993-4517-bccb-7e824426efa8","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:60808","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":33,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:35.461384393Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:47766"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:35.461504174Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"39cd44ab-42c5-4122-917f-987097982291","remote_addr":"[fd00:1122:3344:101::a]:47766","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":46,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:35.52405606Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:35.52409578Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"91574d3b-4803-401d-9d6b-9f2654648006","remote_addr":"[fd00:1122:3344:101::a]:47766","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":79,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:35.537861783Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"2d5a71d2-96aa-4d29-8e90-d3859102d49e","remote_addr":"[fd00:1122:3344:101::a]:47766","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":184,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:35.653024682Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:55294"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:35.653114092Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:35.690166649Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:35.743738948Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"19da0376-2412-4b1a-8ab4-df5fdb1f5d18","remote_addr":"[fd00:1122:3344:101::a]:55294","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":90658,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:35.7439921Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:59752"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:35.744239062Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"ed7e1d39-61c9-42d5-934f-88c996351270","remote_addr":"[fd00:1122:3344:101::a]:55294","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":229,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:35.744323112Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"442bff5d-872c-4ea9-b617-c3fb950c5a2a","remote_addr":"[fd00:1122:3344:101::a]:59752","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":198,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:36.293296097Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:46252"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:36.293394938Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"1b514a3b-c7ec-4f02-8ebd-f35af5fb8d72","remote_addr":"[fd00:1122:3344:101::b]:46252","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":47,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:36.357403225Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:36.357432225Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"58a38209-de3d-418a-80ca-09b01a40f1a6","remote_addr":"[fd00:1122:3344:101::b]:46252","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":56,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:36.370440232Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"7a081567-a7d8-4fb0-ba00-78b987f1cdd3","remote_addr":"[fd00:1122:3344:101::b]:46252","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":265,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:36.395719521Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:49744"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:36.395915152Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:36.43453356Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:36.490077034Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"85d0f92d-3f77-4fbe-97af-69ffb45dc5d9","remote_addr":"[fd00:1122:3344:101::b]:49744","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":94229,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:36.490266166Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:51900"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:36.490599418Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"66439340-e0d8-4aa3-816d-5abcb1d8ab01","remote_addr":"[fd00:1122:3344:101::b]:51900","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":229,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:36.490634789Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"1d330376-f794-4af7-a581-63395d70a924","remote_addr":"[fd00:1122:3344:101::b]:49744","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":253,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:36.558785207Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:46872"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:36.558956238Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"60807f5a-719d-4d13-b262-0ce0f4277eaa","remote_addr":"[fd00:1122:3344:101::c]:46872","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":62,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:36.575734383Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:60272"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:36.575930435Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"dee357f5-0bb4-49ea-bcc3-f59a62f777c9","remote_addr":"[fd00:1122:3344:101::c]:60272","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":132,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:36.601400785Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:36.601419845Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"c362d70a-8dda-43bf-a058-81b9f85746b9","remote_addr":"[fd00:1122:3344:101::c]:46872","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":287,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:36.611218758Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"a34c1352-8382-4d39-859e-a31e3057b4b1","remote_addr":"[fd00:1122:3344:101::c]:46872","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":217,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:36.622662733Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:40332"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:36.622746014Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:36.656550476Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:36.708983507Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"04eb17f1-5fd7-427f-acb8-55d54a9669a8","remote_addr":"[fd00:1122:3344:101::c]:40332","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":86261,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:03:36.709235469Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"b0959298-4b3c-4f7e-ad23-87b72298a7b2","remote_addr":"[fd00:1122:3344:101::c]:40332","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":103,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:00.145633766Z","hostname":"sock","pid":1574,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:35629"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:00.145740637Z","hostname":"sock","pid":1574,"uri":"/baseboard","method":"GET","req_id":"bb078f41-4ffe-49bc-8624-b11b26549817","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:35629","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":30,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:05.767286501Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:40808"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:05.767504503Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"6aae0d63-7290-41bb-91ce-c864be935c2e","remote_addr":"[fd00:1122:3344:101::a]:40808","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":132,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:05.792243938Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:39412"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:05.792313018Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"73160363-6a8e-4629-a948-3b789e35896c","remote_addr":"[fd00:1122:3344:101::a]:39412","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":29,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:05.833811217Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:05.833836088Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"9a4acc10-0524-4607-9e21-40fb255201cd","remote_addr":"[fd00:1122:3344:101::a]:39412","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":55,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:05.843079347Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"60ecbf31-d26f-4343-9016-a3ec889ffb5e","remote_addr":"[fd00:1122:3344:101::a]:39412","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":166,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:05.892886688Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:41398"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:05.892982749Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:05.930790091Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:05.984603022Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"e6b412ca-a6a6-4fc4-a427-c4f34f010cb8","remote_addr":"[fd00:1122:3344:101::a]:41398","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":91649,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:05.984882084Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"bef94d82-6034-4f80-8f4f-830f810dae4b","remote_addr":"[fd00:1122:3344:101::a]:41398","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":120,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:06.378358959Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:42292"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:06.37848171Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"85d53ca3-d729-4e87-ae5f-8a4bd6ff23f9","remote_addr":"[fd00:1122:3344:101::b]:42292","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":58,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:06.422989682Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:06.423011322Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"1f23f63a-7fa0-4a28-ba71-7e4a36658b55","remote_addr":"[fd00:1122:3344:101::b]:42292","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":66,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:06.433377919Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"01abfe69-1e6a-46d6-ae2b-663e8fed6f8c","remote_addr":"[fd00:1122:3344:101::b]:42292","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":152,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:06.453089976Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:55104"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:06.453241617Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"a8e99267-3a31-46ad-9dfd-8e9e0740e252","remote_addr":"[fd00:1122:3344:101::c]:55104","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":40,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:06.514187082Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:06.514221432Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"3ce1d022-1b81-4598-ab3e-e538b056d220","remote_addr":"[fd00:1122:3344:101::c]:55104","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":72,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:06.524319247Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"2e249f29-4c92-4786-8e77-3cd9a0a590e2","remote_addr":"[fd00:1122:3344:101::c]:55104","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":222,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:06.539469231Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:65101"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:06.539659972Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"05857dca-110e-4832-9387-8bb245542708","remote_addr":"[fd00:1122:3344:101::b]:65101","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":131,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:06.552725279Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:46758"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:06.55282865Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:06.58905823Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:06.643867399Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"2873ddac-bdc8-4cd4-a311-b60f75a98fac","remote_addr":"[fd00:1122:3344:101::b]:46758","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":91075,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:06.644135121Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"d76d41b8-99e9-4a66-8329-e26b2805e429","remote_addr":"[fd00:1122:3344:101::b]:46758","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":112,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:06.710041123Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:56855"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:06.710248694Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"7f9bed4c-7ffa-4455-b4db-3de6a9c32c2c","remote_addr":"[fd00:1122:3344:101::c]:56855","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":144,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:06.782337502Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:61019"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:06.782442343Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:06.815222857Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:06.86655593Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"2bff2fd8-645e-4c98-aad4-64be6d9c85c3","remote_addr":"[fd00:1122:3344:101::c]:61019","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":84147,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:06.866865752Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"272e3c60-0d60-4ec0-9acb-8c508b0833e3","remote_addr":"[fd00:1122:3344:101::c]:61019","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":107,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:30.178604279Z","hostname":"sock","pid":1574,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:37078"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:30.178726Z","hostname":"sock","pid":1574,"uri":"/baseboard","method":"GET","req_id":"47391966-86ab-4812-aae4-addab8b09c5c","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:37078","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":35,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:35.519690623Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:37589"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:35.519816233Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"94a0e77f-6a15-48fc-8116-36204e06cb0c","remote_addr":"[fd00:1122:3344:101::a]:37589","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":54,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:35.538701954Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:35.538727734Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"4a529fa3-8497-4e8c-b8c0-b7c9bae4a6ef","remote_addr":"[fd00:1122:3344:101::a]:37589","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":60,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:35.548632919Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"30fee413-528e-42ad-af07-e921ba954306","remote_addr":"[fd00:1122:3344:101::a]:37589","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":144,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:35.634402588Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:47888"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:35.634558419Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"be1bbc4e-e04a-45f9-ac50-0979286731b4","remote_addr":"[fd00:1122:3344:101::a]:47888","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":104,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:35.660248281Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:40732"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:35.660312581Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:35.695728996Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:35.747474901Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"536c5752-f9d7-4a45-a410-39fdcb0abb96","remote_addr":"[fd00:1122:3344:101::a]:40732","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":87189,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:35.747842884Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"0649d8fd-f1e0-4b40-8c3a-506a7107d7b6","remote_addr":"[fd00:1122:3344:101::a]:40732","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":112,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:36.407705255Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:55725"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:36.407870156Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"1100c38e-5d83-4c94-87dd-36f29384d73d","remote_addr":"[fd00:1122:3344:101::b]:55725","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":43,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:36.421615119Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:36.421632529Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"c0e9a109-8e1a-4a2f-849c-699b682e7359","remote_addr":"[fd00:1122:3344:101::b]:55725","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":56,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:36.43108763Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"ca002690-855b-4383-a75f-0e7ea0190b34","remote_addr":"[fd00:1122:3344:101::b]:55725","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":239,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:36.43242756Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:37027"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:36.432631951Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"2890983a-ffa8-45e2-8b26-b2ec3e35b7a0","remote_addr":"[fd00:1122:3344:101::b]:37027","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":109,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:36.470336822Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:57650"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:36.470420573Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:36.509205672Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:36.561695634Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"3f124134-d390-4d3d-a9c7-97876b32544b","remote_addr":"[fd00:1122:3344:101::b]:57650","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":91307,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:36.561846025Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:58400"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:36.562054297Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"b1010dbc-d676-4671-85e0-e9e4fcb6156d","remote_addr":"[fd00:1122:3344:101::c]:58400","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":93,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:36.562231228Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"8d58654b-033c-49e2-ac7c-743416a82fd3","remote_addr":"[fd00:1122:3344:101::b]:57650","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":195,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:36.576822947Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:36.576841817Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"8a9b68d3-bc7e-4380-a0bb-5b1cd0ba23f4","remote_addr":"[fd00:1122:3344:101::c]:58400","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":88,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:36.586133636Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"0b4b3025-d38b-4f6e-90b1-fffe4cf8003e","remote_addr":"[fd00:1122:3344:101::c]:58400","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":224,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:36.656149308Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:55006"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:36.6563146Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"eb5577c9-cfe1-4bb7-951c-17294fd2bd64","remote_addr":"[fd00:1122:3344:101::c]:55006","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":102,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:36.699514052Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:40561"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:36.699615183Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:36.736805249Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:36.790749342Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"7c11bb76-3daa-4141-ab03-1022cb6afd71","remote_addr":"[fd00:1122:3344:101::c]:40561","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":91189,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:04:36.791097544Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"ae6eb248-33bf-433b-86e0-4914106d8b48","remote_addr":"[fd00:1122:3344:101::c]:40561","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":108,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:00.211320211Z","hostname":"sock","pid":1574,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:36946"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:00.211464482Z","hostname":"sock","pid":1574,"uri":"/baseboard","method":"GET","req_id":"93cce9d8-4748-48d5-b41c-2b10eb4b549c","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:36946","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":43,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:05.677064334Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:62721"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:05.677192535Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"0d9c10e9-0ca8-4a62-835b-c4eca7fdf61e","remote_addr":"[fd00:1122:3344:101::a]:62721","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":53,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:05.692886482Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:05.692903322Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"b45265f7-8531-4c3b-8d24-95b2fdb1c104","remote_addr":"[fd00:1122:3344:101::a]:62721","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":54,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:05.845672931Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:51361"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:05.845841272Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"3ec78839-b7a0-4709-80ff-319177b90055","remote_addr":"[fd00:1122:3344:101::a]:51361","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":109,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:05.849294788Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"e7116faa-fd13-4336-aa5c-9577ae564e26","remote_addr":"[fd00:1122:3344:101::a]:62721","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":153,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:06.065880874Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:41593"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:06.065975574Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:06.099592485Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:06.18604662Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"0fb7f93a-d4e1-45cc-9f74-f1e9142837d1","remote_addr":"[fd00:1122:3344:101::a]:41593","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":120095,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:06.186509933Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"3f86a723-697e-4a66-99da-28755fa235b1","remote_addr":"[fd00:1122:3344:101::a]:41593","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":175,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:06.336872515Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:55252"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:06.337038996Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"b3d9916f-8174-407c-99b2-1ea9222f5aae","remote_addr":"[fd00:1122:3344:101::b]:55252","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":91,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:06.398466024Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:06.398501874Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"674d140c-cafa-4abb-b038-0a12149cdfa9","remote_addr":"[fd00:1122:3344:101::b]:55252","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":101,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:06.407737243Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"0104f4d4-ed86-4008-a2ca-635413bd3826","remote_addr":"[fd00:1122:3344:101::b]:55252","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":203,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:06.453034051Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:49404"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:06.453135042Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"4811f21b-1f92-401b-b106-8e9f4f95ab7c","remote_addr":"[fd00:1122:3344:101::c]:49404","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":64,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:06.506773742Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:06.506836622Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"21d587ce-fe7c-47b9-bf14-3553ae22d616","remote_addr":"[fd00:1122:3344:101::c]:49404","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":117,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:06.524883827Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"f4c57d9a-393c-4149-83d2-18ae4647d7f3","remote_addr":"[fd00:1122:3344:101::c]:49404","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":250,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:06.548814125Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:42345"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:06.548928046Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:06.590124493Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:06.648276397Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"b2ef3038-529e-48d8-9f38-55f3eb5857dc","remote_addr":"[fd00:1122:3344:101::b]:42345","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":99389,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:06.648418908Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:63319"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:06.648755551Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"99656427-da73-4871-a45b-23d5089ef094","remote_addr":"[fd00:1122:3344:101::b]:63319","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":219,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:06.648862122Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"4b42aa72-919b-4001-b3f4-000068800345","remote_addr":"[fd00:1122:3344:101::b]:42345","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":250,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:06.653950809Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:63883"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:06.654156871Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"468d9836-f0a5-46a1-b6fb-7836ebc563cf","remote_addr":"[fd00:1122:3344:101::c]:63883","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":136,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:06.804834094Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:61809"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:06.804959565Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:06.839236011Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:06.895866223Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"9cca91f1-7b63-4951-a440-6e98ac624012","remote_addr":"[fd00:1122:3344:101::c]:61809","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":90966,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:06.896196946Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"920757aa-c6b6-4cb8-b5f2-868f95a3addb","remote_addr":"[fd00:1122:3344:101::c]:61809","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":118,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:30.243541758Z","hostname":"sock","pid":1574,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:63379"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:30.243646109Z","hostname":"sock","pid":1574,"uri":"/baseboard","method":"GET","req_id":"3b547b09-9eee-4a7e-bffb-549775993ac3","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:63379","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":30,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:35.712983779Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:59201"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:35.71310708Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"6b7969f1-df2f-428b-8945-dc37e808d713","remote_addr":"[fd00:1122:3344:101::a]:59201","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":52,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:35.763449465Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:35.763482996Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"a334404c-3a33-4b56-9e4c-049f4f3bd7df","remote_addr":"[fd00:1122:3344:101::a]:59201","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":77,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:35.772054929Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"72b6daad-e5f5-44ad-921a-2aacaf2909f2","remote_addr":"[fd00:1122:3344:101::a]:59201","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":176,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:35.798489897Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:63161"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:35.798634288Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"c1bb2c68-bd36-445d-afcc-20c00fc2ee65","remote_addr":"[fd00:1122:3344:101::a]:63161","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":107,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:35.813925472Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:59986"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:35.814007752Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:35.850310713Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:35.901921918Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"c2b8084a-d093-4142-ace3-6ffd623bbbce","remote_addr":"[fd00:1122:3344:101::a]:59986","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":87948,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:35.90216437Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"0db23ab9-02ef-4391-9fe0-28f37e9bd650","remote_addr":"[fd00:1122:3344:101::a]:59986","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":110,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:36.344534229Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:38710"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:36.34465867Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"84a415b6-4aab-4b77-a328-50da498fca17","remote_addr":"[fd00:1122:3344:101::b]:38710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":55,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:36.361313194Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:36.361335564Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"b7434dd9-5d4f-4793-b7f6-19dd146a2d5f","remote_addr":"[fd00:1122:3344:101::b]:38710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":105,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:36.374449282Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"92d10b4c-8ecd-46dc-91b7-335f33bb998d","remote_addr":"[fd00:1122:3344:101::b]:38710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":241,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:36.458259727Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:37961"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:36.458342738Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"b3ebd837-caea-4e21-bf62-f53d12dd68f0","remote_addr":"[fd00:1122:3344:101::c]:37961","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":42,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:36.461062358Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:48847"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:36.46126671Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"8dcf176e-4b60-413c-bb0a-7e75121c0695","remote_addr":"[fd00:1122:3344:101::b]:48847","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":115,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:36.49483525Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:49051"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:36.494918421Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:36.529351988Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:36.5873546Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"1e6e5624-5e32-459a-80a8-b125ee7d001e","remote_addr":"[fd00:1122:3344:101::b]:49051","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":92466,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:36.587515671Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:48082"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:36.587574272Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:35821"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:36.587814873Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:36.587844604Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"6c85be70-93d6-4c38-b4c2-7da32f9b572e","remote_addr":"[fd00:1122:3344:101::c]:37961","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":186,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:36.587934054Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"037fa5ab-b20e-4b2e-8ece-8dacecf3b8e2","remote_addr":"[fd00:1122:3344:101::b]:49051","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":286,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:36.587969525Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:36.588018955Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"04483a40-6ce7-4c8c-a4f5-31fc7d6e2114","remote_addr":"[fd00:1122:3344:101::c]:48082","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":321,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:36.597212634Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"cb22acae-dafc-41f1-a7dd-a26ebe87784a","remote_addr":"[fd00:1122:3344:101::c]:37961","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":254,"response_code":"204"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:36.626481442Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:36.68112218Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"1bcb405b-a8e9-4a7b-bc58-bc5aebda9c4c","remote_addr":"[fd00:1122:3344:101::c]:35821","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":93300,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:05:36.693680133Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"d166007a-4358-4469-a61a-ad50ccc39037","remote_addr":"[fd00:1122:3344:101::c]:35821","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":110,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:00.276034608Z","hostname":"sock","pid":1574,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:61297"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:00.276174949Z","hostname":"sock","pid":1574,"uri":"/baseboard","method":"GET","req_id":"c7098c12-d69b-4761-b294-da584a988b0b","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:61297","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":42,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:05.627262207Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:55373"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:05.627389488Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"6c94718f-bc46-4703-a933-d845f8a05c7e","remote_addr":"[fd00:1122:3344:101::a]:55373","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":56,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:05.672721076Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:05.672755706Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"b63e16c3-560e-4a25-8bd1-c103b25b98ea","remote_addr":"[fd00:1122:3344:101::a]:55373","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":75,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:05.68932936Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"1d6cec7c-6e47-42bb-86d1-7b6505ee5ebb","remote_addr":"[fd00:1122:3344:101::a]:55373","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":201,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:05.841230603Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:59114"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:05.841429394Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"a49879a0-18db-40cc-9fbd-46b5088d0a2d","remote_addr":"[fd00:1122:3344:101::a]:59114","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":135,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:05.889466082Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:59643"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:05.889560633Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:05.931294114Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:05.989292827Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"7141759d-ea4e-416c-9364-4138efb34344","remote_addr":"[fd00:1122:3344:101::a]:59643","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":99770,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:05.98973832Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"a08b810e-66d4-44ad-9312-a2b323133183","remote_addr":"[fd00:1122:3344:101::a]:59643","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":200,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:06.035101219Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:62289"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:06.035347321Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3b3dea8a-e84f-4acf-bafc-3b0e4a815dc5, kind: Service { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 }, name: Name(\"external-dns-be35e232-dd9b-4d59-b4e1-eadb7f69bb77\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 109])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 95dce781-7d6b-4bf1-88f1-880b60d0adba, kind: Service { id: 14e637a2-51ce-4cac-b831-85b757e5c55d }, name: Name(\"external-dns-14e637a2-51ce-4cac-b831-85b757e5c55d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 238])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 5dbd0a76-35d9-4b29-8800-824968d1caba, kind: Service { id: 9b2d2174-b236-4546-bace-300e05afe3c2 }, name: Name(\"nexus-9b2d2174-b236-4546-bace-300e05afe3c2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 121])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: db0c0ea5-0d8f-4f9d-9979-f3989ce962ea, kind: Service { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 }, name: Name(\"nexus-16a7178a-dbb6-4392-b395-7bddb3817d13\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e67bea26-1a67-4166-875f-3cab431644e6, kind: Service { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 }, name: Name(\"nexus-3dce42ed-74cb-4c8b-9326-8273a1efd563\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 239])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:06.035400431Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:06.035438151Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:06.035456881Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:06.035468671Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:06.035498522Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:06.035510332Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:06.035531512Z","hostname":"sock","pid":1574,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"9c15aade-5ba2-4597-b68a-6262e0355bd3","remote_addr":"[fd00:1122:3344:101::a]:62289","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":366,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:06.436642023Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:43524"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:06.436839105Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"89033fc5-126e-4217-9535-af91d2fccf96","remote_addr":"[fd00:1122:3344:101::b]:43524","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":130,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:06.549898938Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:50770"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:06.549984019Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"33528fd9-34bf-477f-93cb-203e1ee99f73","remote_addr":"[fd00:1122:3344:101::b]:50770","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":37,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:06.592222804Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:06.592242404Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"63fa2dbd-9041-4ccd-ab4b-28b410e9031b","remote_addr":"[fd00:1122:3344:101::b]:50770","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":55,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:06.601649184Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"c0883592-f7f4-44cf-aa5b-6e0e3cdec9e3","remote_addr":"[fd00:1122:3344:101::b]:50770","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":152,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:06.615611128Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:44366"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:06.615670549Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:06.654138825Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:06.716169648Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"ec4605a6-c894-4b3f-870c-a381c1a3f0ee","remote_addr":"[fd00:1122:3344:101::b]:44366","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":100519,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:06.716300739Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:38324"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:06.716349849Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:37969"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:06.716626191Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"dc8ed7a1-9b96-471b-ab0c-f438c52d22ba","remote_addr":"[fd00:1122:3344:101::c]:37969","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":165,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:06.716681062Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"2fdad7e9-f4ee-4993-b232-217211ba3d4f","remote_addr":"[fd00:1122:3344:101::c]:38324","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":117,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:06.716715512Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"c062e785-d18f-4920-9c23-65d1bde26622","remote_addr":"[fd00:1122:3344:101::b]:44366","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":237,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:06.731818244Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:06.731845115Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"de29bb4d-90d1-4f9a-b1a4-99458f487bec","remote_addr":"[fd00:1122:3344:101::c]:38324","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":97,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:06.740304318Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"b57634c6-c1cd-4f1a-bd84-99d04945e1a1","remote_addr":"[fd00:1122:3344:101::c]:38324","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":227,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:06.788482047Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:49665"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:06.788654619Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:06.821144221Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:06.875004783Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"95ca482f-4e99-487a-9215-dd4ad5047c47","remote_addr":"[fd00:1122:3344:101::c]:49665","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":86400,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:06.875154484Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:59345"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:06.875208484Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:44131"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:06.875509766Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"ad94c9a6-de60-4c94-9bcd-167fa16bcdb2","remote_addr":"[fd00:1122:3344:101::c]:49665","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":242,"response_code":"200"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:06.875666087Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3b3dea8a-e84f-4acf-bafc-3b0e4a815dc5, kind: Service { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 }, name: Name(\"external-dns-be35e232-dd9b-4d59-b4e1-eadb7f69bb77\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 109])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 95dce781-7d6b-4bf1-88f1-880b60d0adba, kind: Service { id: 14e637a2-51ce-4cac-b831-85b757e5c55d }, name: Name(\"external-dns-14e637a2-51ce-4cac-b831-85b757e5c55d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 238])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 5dbd0a76-35d9-4b29-8800-824968d1caba, kind: Service { id: 9b2d2174-b236-4546-bace-300e05afe3c2 }, name: Name(\"nexus-9b2d2174-b236-4546-bace-300e05afe3c2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 121])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: db0c0ea5-0d8f-4f9d-9979-f3989ce962ea, kind: Service { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 }, name: Name(\"nexus-16a7178a-dbb6-4392-b395-7bddb3817d13\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e67bea26-1a67-4166-875f-3cab431644e6, kind: Service { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 }, name: Name(\"nexus-3dce42ed-74cb-4c8b-9326-8273a1efd563\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 239])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:06.875717918Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:06.875751378Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:06.875771148Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:06.875779318Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:06.875790328Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:06.875808449Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:06.875814169Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3b3dea8a-e84f-4acf-bafc-3b0e4a815dc5, kind: Service { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 }, name: Name(\"external-dns-be35e232-dd9b-4d59-b4e1-eadb7f69bb77\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 109])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 95dce781-7d6b-4bf1-88f1-880b60d0adba, kind: Service { id: 14e637a2-51ce-4cac-b831-85b757e5c55d }, name: Name(\"external-dns-14e637a2-51ce-4cac-b831-85b757e5c55d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 238])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 5dbd0a76-35d9-4b29-8800-824968d1caba, kind: Service { id: 9b2d2174-b236-4546-bace-300e05afe3c2 }, name: Name(\"nexus-9b2d2174-b236-4546-bace-300e05afe3c2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 121])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: db0c0ea5-0d8f-4f9d-9979-f3989ce962ea, kind: Service { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 }, name: Name(\"nexus-16a7178a-dbb6-4392-b395-7bddb3817d13\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e67bea26-1a67-4166-875f-3cab431644e6, kind: Service { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 }, name: Name(\"nexus-3dce42ed-74cb-4c8b-9326-8273a1efd563\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 239])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:06.875834829Z","hostname":"sock","pid":1574,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"33639b6e-1b06-4d7f-92b8-e1fb49c68160","remote_addr":"[fd00:1122:3344:101::b]:59345","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":496,"response_code":"204"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:06.875883209Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:06.875923439Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:06.87595301Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:06.87596463Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:06.87599522Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:06.87601417Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:06.87604074Z","hostname":"sock","pid":1574,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"6b6037bf-e605-4279-ae43-2fee52f8513f","remote_addr":"[fd00:1122:3344:101::c]:44131","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":574,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:30.309074102Z","hostname":"sock","pid":1574,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:57954"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:30.309220573Z","hostname":"sock","pid":1574,"uri":"/baseboard","method":"GET","req_id":"f3e2db17-b1f0-4d67-9a87-66e3d3bec17b","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:57954","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":43,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:35.535264388Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:61958"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:35.535419059Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"f768b50d-dff8-4876-936f-434c07c264c5","remote_addr":"[fd00:1122:3344:101::a]:61958","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":76,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:35.577443243Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:35.577484073Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"f3f947e9-b5b2-48cb-abad-96dfec7b5853","remote_addr":"[fd00:1122:3344:101::a]:61958","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":62,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:35.586592111Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"22a360ae-2595-4f4c-a211-adfaf9c6bdd9","remote_addr":"[fd00:1122:3344:101::a]:61958","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":194,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:35.620793696Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:41260"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:35.620890227Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:35.659010321Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:35.715624433Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"71be3dc5-84df-493b-88fa-d0fe182ccc3b","remote_addr":"[fd00:1122:3344:101::a]:41260","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":94775,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:35.715744624Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:52379"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:35.716101687Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"fbce2c6a-dbc6-4418-8b03-35f64af27fd4","remote_addr":"[fd00:1122:3344:101::a]:52379","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":196,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:35.716163237Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"51d851be-83c4-43c8-8296-e02c3f05c138","remote_addr":"[fd00:1122:3344:101::a]:41260","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":246,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:36.352770305Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:51055"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:36.352856866Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"988ba860-4963-49fc-ab61-7a174387c6ab","remote_addr":"[fd00:1122:3344:101::c]:51055","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":45,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:36.36685688Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:36.366877051Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"06ebff77-220d-4bae-a6b3-c0fbc82e3e3e","remote_addr":"[fd00:1122:3344:101::c]:51055","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":64,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:36.375427494Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"0026d567-0b73-41ec-90a3-39869d6365dc","remote_addr":"[fd00:1122:3344:101::c]:51055","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":212,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:36.378453617Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:33596"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:36.378582818Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"edef37fd-ed9d-4bc2-a478-950c0f7c1e09","remote_addr":"[fd00:1122:3344:101::b]:33596","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":35,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:36.422320414Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:36.422352554Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"7fa2fd62-92cb-4dac-89d4-5a285119ecd8","remote_addr":"[fd00:1122:3344:101::b]:33596","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":50,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:36.431853955Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"a5fae99a-6faf-488b-8a29-65332451804a","remote_addr":"[fd00:1122:3344:101::b]:33596","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":145,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:36.46465082Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:41378"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:36.46476061Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:36.499980493Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:36.551692809Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"c8d0c9f8-904f-4e7a-86cd-5256d4419892","remote_addr":"[fd00:1122:3344:101::b]:41378","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":86986,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:36.551937151Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:63542"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:36.552040681Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:56234"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:36.552103542Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:51972"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:36.552311714Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"6d6e9593-c92d-4d51-bc4d-15e33d828dba","remote_addr":"[fd00:1122:3344:101::b]:41378","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":319,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:36.552434644Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:36.552475055Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"c80e32c6-4820-4469-a97a-53ff07b3365c","remote_addr":"[fd00:1122:3344:101::c]:63542","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":224,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:36.56252074Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"efa227eb-43d1-44a1-8d68-776d1b857e2f","remote_addr":"[fd00:1122:3344:101::b]:56234","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":10101,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:36.589270159Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:36.643456023Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"7fe4b251-9915-4784-84e7-ad4d8243f3e1","remote_addr":"[fd00:1122:3344:101::c]:51972","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":91161,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:06:36.643791946Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"f2dddbc5-72f1-4f1c-bcee-2a78699fb9d1","remote_addr":"[fd00:1122:3344:101::c]:51972","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":141,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:00.342050725Z","hostname":"sock","pid":1574,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:40000"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:00.342151566Z","hostname":"sock","pid":1574,"uri":"/baseboard","method":"GET","req_id":"1ab7bdaf-c385-4884-8796-be7bbb05d490","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:40000","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":33,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:05.728107204Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:52634"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:05.728227394Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"9ed46a62-4932-4045-9515-7e59e7219abf","remote_addr":"[fd00:1122:3344:101::a]:52634","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":51,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:05.770657261Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:05.770704361Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"237f9ddc-dc66-4cc0-b439-2749ed9f88c3","remote_addr":"[fd00:1122:3344:101::a]:52634","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":73,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:05.779512327Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"537ea45a-4f3e-408b-b138-cd2e65a61387","remote_addr":"[fd00:1122:3344:101::a]:52634","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":165,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:05.842851949Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:51219"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:05.84299959Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"f27d67b8-2559-4108-bfcc-be391b0f35d1","remote_addr":"[fd00:1122:3344:101::a]:51219","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":114,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:05.857973022Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:52234"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:05.858057953Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:05.900717481Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:05.953094431Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"222ad1ea-45f5-481c-b82f-496e2588ddda","remote_addr":"[fd00:1122:3344:101::a]:52234","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":95068,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:05.953388314Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"8092905b-13e5-44f6-b1c4-07df7df754a1","remote_addr":"[fd00:1122:3344:101::a]:52234","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":105,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:06.438979535Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:49006"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:06.439141647Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"dbb9219e-f00a-415b-a336-1e3797153c86","remote_addr":"[fd00:1122:3344:101::b]:49006","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":59,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:06.498449919Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:06.498483449Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"8ed69994-a839-4c83-90aa-fe93c74447e2","remote_addr":"[fd00:1122:3344:101::b]:49006","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":81,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:06.507840099Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"2bf84cee-21ce-4606-877d-36b87f869539","remote_addr":"[fd00:1122:3344:101::b]:49006","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":180,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:06.593215236Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:45797"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:06.593322066Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:06.633848519Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:06.69437968Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"1c80c25b-ff01-4346-b348-a73adf4cfb6b","remote_addr":"[fd00:1122:3344:101::b]:45797","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":101105,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:06.694492851Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:58099"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:06.694529571Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:39166"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:06.694859884Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"a9503d94-8749-4f03-b199-17d22bf7b354","remote_addr":"[fd00:1122:3344:101::b]:39166","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":251,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:06.694891764Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"581747b6-0687-463d-9c93-fdc2193a0916","remote_addr":"[fd00:1122:3344:101::c]:58099","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":128,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:06.694914614Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"cc4ded71-30a3-46d5-ac24-fd640013a0fb","remote_addr":"[fd00:1122:3344:101::b]:45797","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":256,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:06.710282738Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:06.710300969Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"b5702c5d-5523-4f31-b290-6b1b1338b778","remote_addr":"[fd00:1122:3344:101::c]:58099","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":117,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:06.727231915Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:54389"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:06.727403876Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"2f998a39-0203-40d7-9339-da632090ccda","remote_addr":"[fd00:1122:3344:101::c]:54389","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":134,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:06.730042346Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"6c65df37-0f13-43ac-93eb-fda0b740ade3","remote_addr":"[fd00:1122:3344:101::c]:58099","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":190,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:06.793402138Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:59938"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:06.793490329Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:06.833068284Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:06.893114752Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"c5d7a8dc-3918-4f6e-adcc-0c1c8c58db14","remote_addr":"[fd00:1122:3344:101::c]:59938","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":99655,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:06.893432155Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"e0eb2fbb-1f57-4491-b783-2d2c7bb42692","remote_addr":"[fd00:1122:3344:101::c]:59938","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":107,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:30.375598433Z","hostname":"sock","pid":1574,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:50344"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:30.375885345Z","hostname":"sock","pid":1574,"uri":"/baseboard","method":"GET","req_id":"2d6850b0-22cd-4e4d-8484-79ec2d2ef9e7","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:50344","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":44,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:35.539273763Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:38838"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:35.539391224Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"145df34a-088b-4d93-be64-9695b1bd437d","remote_addr":"[fd00:1122:3344:101::a]:38838","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":53,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:35.548432951Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:36927"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:35.548593502Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"1c7370d2-7555-456e-9d3f-8102877e279c","remote_addr":"[fd00:1122:3344:101::a]:36927","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":110,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:35.554347545Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:35.554363225Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"99dbafa9-dea2-4190-9404-ba3d32858e6b","remote_addr":"[fd00:1122:3344:101::a]:38838","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":60,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:35.56439997Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"0a2d0eec-1f50-4905-b10e-23fe625b2507","remote_addr":"[fd00:1122:3344:101::a]:38838","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":177,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:35.598413354Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:35380"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:35.598489964Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:35.638582323Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:35.699485858Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"2c397284-e348-4557-8cec-b8de8d4e5c08","remote_addr":"[fd00:1122:3344:101::a]:35380","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":101018,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:35.699891151Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"6a3f3a8d-c311-4a65-8213-15f0680be906","remote_addr":"[fd00:1122:3344:101::a]:35380","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":177,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:36.351559781Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:38697"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:36.351689701Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"3f248b4a-a075-4061-9181-23319df5af9d","remote_addr":"[fd00:1122:3344:101::b]:38697","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":52,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:36.401824315Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:36.401850766Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"3d2f0fd2-a0fa-4857-a32c-9ddac9c422bf","remote_addr":"[fd00:1122:3344:101::b]:38697","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":85,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:36.411482307Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"a89e310f-57f3-4ee6-a119-6d9881a83461","remote_addr":"[fd00:1122:3344:101::b]:38697","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":159,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:36.503345873Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:56898"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:36.503540494Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"f7b42421-ed37-45f9-84bf-a3a20421c6d7","remote_addr":"[fd00:1122:3344:101::b]:56898","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":127,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:36.535355942Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:60224"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:36.535464162Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:36.573676857Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:36.631303937Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"87ef9e4b-1e24-4e63-b5e0-21ab75385889","remote_addr":"[fd00:1122:3344:101::b]:60224","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":95878,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:36.631423288Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:58681"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:36.631462968Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:51908"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:36.63173799Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"a277feb5-011d-4111-94b0-849196a290aa","remote_addr":"[fd00:1122:3344:101::c]:51908","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":205,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:36.631813261Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"2f600de3-d572-4342-aca4-26a63df27cce","remote_addr":"[fd00:1122:3344:101::c]:58681","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":171,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:36.631913271Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"e4260596-8b21-496b-bfb8-0b347e69d872","remote_addr":"[fd00:1122:3344:101::b]:60224","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":268,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:36.648490755Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:36.648505585Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"c01ca75d-263b-4d4d-bc7f-81487914eae5","remote_addr":"[fd00:1122:3344:101::c]:58681","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":82,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:36.658131257Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"23880b7b-465b-4d5d-a93b-9e4ce54fc67e","remote_addr":"[fd00:1122:3344:101::c]:58681","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":186,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:36.680987697Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:38592"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:36.681087368Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:36.717839572Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:36.774130292Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"e710110c-474d-477d-9968-100806bd237e","remote_addr":"[fd00:1122:3344:101::c]:38592","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":93079,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:07:36.774419014Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"d7a4a5c4-e296-4172-80b9-5565ad74db53","remote_addr":"[fd00:1122:3344:101::c]:38592","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":102,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:00.416117442Z","hostname":"sock","pid":1574,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:53953"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:00.416276423Z","hostname":"sock","pid":1574,"uri":"/baseboard","method":"GET","req_id":"d0f6596c-3c13-4b50-a08e-910859329010","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:53953","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":81,"response_code":"200"} {"msg":"running automatic periodic zone bundle cleanup","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:03.301522541Z","hostname":"sock","pid":1574,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:197"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:03.301592211Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:03.332377081Z","hostname":"sock","pid":1574,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:186","time_to_next_cleanup":"599.9999957s","next_cleanup":"Instant { tv_sec: 1992, tv_nsec: 645573396 }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:05.641078849Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:42467"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:05.64123632Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"854cfd10-b395-443b-bb49-398ce664fd22","remote_addr":"[fd00:1122:3344:101::a]:42467","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":80,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:05.773541897Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:36682"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:05.773701168Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"6db967ee-6bbc-4e58-bb67-76bec8a01336","remote_addr":"[fd00:1122:3344:101::a]:36682","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":107,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:05.828699088Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:05.828727498Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"d4340b0e-2ff0-4d02-bd07-c692e1f15d4c","remote_addr":"[fd00:1122:3344:101::a]:42467","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":64,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:05.837726836Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"cd3c1acb-ff94-4ffb-8cfa-911af263b1a1","remote_addr":"[fd00:1122:3344:101::a]:42467","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":149,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:05.964766693Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:55403"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:05.964852854Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:06.002658796Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:06.05662706Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"99a0535b-0eb3-4e1b-ba9d-cb480f7632ed","remote_addr":"[fd00:1122:3344:101::a]:55403","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":91868,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:06.057108802Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"aee72a44-53ce-4b78-93a5-85526ade3dab","remote_addr":"[fd00:1122:3344:101::a]:55403","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":202,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:06.423940658Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:58578"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:06.424043069Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"daf94595-692e-4d0d-8523-147907d7b5e2","remote_addr":"[fd00:1122:3344:101::b]:58578","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":46,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:06.443302442Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:06.443324662Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"cc89aebc-6139-4724-b38a-01a5527ea3f3","remote_addr":"[fd00:1122:3344:101::b]:58578","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":63,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:06.512222306Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:33100"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:06.512410678Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"3e4a5c5a-6f79-49a9-8453-1ee98ed024ca","remote_addr":"[fd00:1122:3344:101::b]:33100","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":136,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:06.518404532Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"76fa4c9a-c2b3-45f1-93f3-7e2391dcfcfc","remote_addr":"[fd00:1122:3344:101::b]:58578","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":221,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:06.584848128Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:54331"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:06.584918169Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"dc99a2b0-bc7b-439c-8eb3-db520fea668b","remote_addr":"[fd00:1122:3344:101::c]:54331","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":30,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:06.594419949Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:47931"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:06.59450104Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:06.631942709Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:06.692627102Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"1b00feb5-eb18-4c9c-ab7d-12e0dda0445e","remote_addr":"[fd00:1122:3344:101::b]:47931","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":98144,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:06.692931194Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:45897"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:06.696567531Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:06.697051455Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"6ccd1d7e-bee3-42a8-9a82-1f5de670bf08","remote_addr":"[fd00:1122:3344:101::c]:54331","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":4139,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:06.697372197Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"1c0637c1-5a36-48dc-b4ee-f80cb473766e","remote_addr":"[fd00:1122:3344:101::c]:45897","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":583,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:06.697439008Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"2e004f62-ec22-417f-b29a-dcf265b01407","remote_addr":"[fd00:1122:3344:101::b]:47931","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":4496,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:06.706444715Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"5b440b4e-1e12-47e1-bf27-ca7a70906b2f","remote_addr":"[fd00:1122:3344:101::c]:54331","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":286,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:06.773563845Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:36300"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:06.773673896Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:06.812048882Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:06.868489163Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"631d19cf-5562-470a-916e-bf99a439d31c","remote_addr":"[fd00:1122:3344:101::c]:36300","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":94860,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:06.868746955Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"6b6f1909-5f3b-42ea-8adf-1e324c2b112f","remote_addr":"[fd00:1122:3344:101::c]:36300","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":94,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:17.878486325Z","hostname":"sock","pid":1574,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:20.078644753Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":4,"n_current_samples":498,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:20.081476165Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":4,"n_current_samples":498,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:20.082667923Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":4,"n_current_samples":498,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:20.083777152Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":4,"n_current_samples":498,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:20.114588812Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":4,"n_current_samples":498,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:20.11570901Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":4,"n_current_samples":498,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:20.119502298Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":4,"n_current_samples":498,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:20.120205603Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":4,"n_current_samples":498,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:20.120915609Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":4,"n_current_samples":498,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:29.865772425Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:29.866700662Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:29.941000746Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:29.941869322Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:30.034008559Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:30.080430926Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:30.081171111Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:30.082769073Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:30.084348995Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:30.08503483Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:30.085571384Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:30.086258479Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:30.116113792Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:30.116636706Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:30.117488862Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:30.121110629Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:30.121653783Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:30.123183785Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:30.449426148Z","hostname":"sock","pid":1574,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:53735"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:30.44964001Z","hostname":"sock","pid":1574,"uri":"/baseboard","method":"GET","req_id":"ab8bb239-07ba-46f2-93bb-ecd57e9d1a4e","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:53735","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":72,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:35.533395304Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:35824"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:35.533527265Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"2c9fd27a-9255-40ad-b512-b09aff30a461","remote_addr":"[fd00:1122:3344:101::a]:35824","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":58,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:35.549971577Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:35.549991068Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"be34209d-a0d6-4a4f-acfa-a44d04f317fc","remote_addr":"[fd00:1122:3344:101::a]:35824","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":58,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:35.559844651Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"aa3d3577-d10e-4635-88a1-7127f16381fc","remote_addr":"[fd00:1122:3344:101::a]:35824","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":185,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:35.656017419Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:48798"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:35.65619001Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"81449376-df33-47a0-a5d9-067b4c32aa18","remote_addr":"[fd00:1122:3344:101::a]:48798","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":110,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:35.683591794Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:42711"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:35.683679295Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:35.726800416Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:35.784692258Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"7b3a9654-17d1-4884-8385-e8d7f1ad3321","remote_addr":"[fd00:1122:3344:101::a]:42711","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":101042,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:35.78768545Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"ae0880e6-7d79-41d7-a0a8-43e307e5bc5e","remote_addr":"[fd00:1122:3344:101::a]:42711","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":84,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:36.298133477Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:41496"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:36.298223038Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"ecddd464-415a-43ba-9983-6aa8f8a9fbb7","remote_addr":"[fd00:1122:3344:101::b]:41496","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":44,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:36.350417417Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:36.350451387Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"f6776afd-d9c1-4dd9-9463-92ee2181f4f9","remote_addr":"[fd00:1122:3344:101::b]:41496","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":61,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:36.359717527Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"bdf47716-f649-4fbc-be3f-2a9971af3c50","remote_addr":"[fd00:1122:3344:101::b]:41496","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":198,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:36.445242474Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:50429"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:36.445347925Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"ab1c6d71-0ba7-4bcc-8905-795936dd9751","remote_addr":"[fd00:1122:3344:101::c]:50429","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":28,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:36.497526994Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:36.497594475Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"00f323b4-2518-4e79-bbfc-729678f0df4e","remote_addr":"[fd00:1122:3344:101::c]:50429","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":110,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:36.504408015Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:59603"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:36.504628827Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"913359fa-78e3-4d5e-9c62-2a75dbe00fa8","remote_addr":"[fd00:1122:3344:101::b]:59603","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":134,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:36.507178176Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"1a4812da-eab3-481f-89ed-832937d3e131","remote_addr":"[fd00:1122:3344:101::c]:50429","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":199,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:36.528791857Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:56542"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:36.528895128Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:36.570017135Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:36.628110458Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"90364fbd-d030-4924-a7b7-df2f9b5b5c30","remote_addr":"[fd00:1122:3344:101::b]:56542","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":99236,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:36.62828839Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:38137"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:36.62833888Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:63241"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:36.628563941Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:36.628604472Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"ad0d2628-f1c5-4c9c-8bdf-59e5a3d34a52","remote_addr":"[fd00:1122:3344:101::b]:56542","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":237,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:36.63235167Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"339faa60-69c0-4ebd-a8e6-76c1c163be71","remote_addr":"[fd00:1122:3344:101::c]:63241","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":3758,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:36.662594865Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:36.717910278Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"632526fd-9c28-42fe-a038-c554192016a1","remote_addr":"[fd00:1122:3344:101::c]:38137","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":89443,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:08:36.718270505Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"19f6dc56-7d85-49fb-ad7c-b52bafe81f30","remote_addr":"[fd00:1122:3344:101::c]:38137","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":153,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:39.868860817Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:39.87068492Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:39.943762746Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:39.944654202Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:40.036996031Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:40.08252394Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:40.083307306Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:40.083873651Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:40.086692342Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:40.08784606Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:40.088412495Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:40.08909282Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:40.11867036Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:40.119225224Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:40.11992085Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:40.123445126Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:40.12398666Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:40.125720433Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:49.872648065Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:49.875106503Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:49.945315827Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:49.946135983Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:50.039266637Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:50.084285133Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:50.085081819Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:50.085669973Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:50.088358353Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:50.089105639Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:50.090693851Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:50.091373396Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:50.12008799Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:50.121693822Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:50.122511728Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:50.125208718Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:50.125802683Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:50.127371454Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:59.876673194Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:59.878332706Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:59.947664083Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:08:59.948442939Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:00.04247377Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:00.086800471Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:00.087524426Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:00.088221231Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:00.091269454Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:00.091940219Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:00.092598004Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:00.093241749Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:00.121825972Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:00.123541415Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:00.12420366Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:00.12695021Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:00.127483424Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:00.129071536Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:00.482299571Z","hostname":"sock","pid":1574,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:40875"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:00.482466242Z","hostname":"sock","pid":1574,"uri":"/baseboard","method":"GET","req_id":"5646afaa-5fa0-4b67-9643-58850f12e3f8","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:40875","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":46,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:05.531822679Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:46123"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:05.5319566Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"78dfef97-fd79-478d-a5ac-ca4eb835c31a","remote_addr":"[fd00:1122:3344:101::a]:46123","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":45,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:05.603631255Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:05.603666806Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"7e0e1f37-fb58-44fd-94c6-feb6c574eaae","remote_addr":"[fd00:1122:3344:101::a]:46123","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":80,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:05.613762851Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"fa8290d4-5ef4-4f5c-9f20-df1b25ba053d","remote_addr":"[fd00:1122:3344:101::a]:46123","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":226,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:05.891489092Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:47178"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:05.891641213Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"f3e1e544-1573-4ee6-94e1-64291ffb1910","remote_addr":"[fd00:1122:3344:101::a]:47178","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":116,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:05.940472317Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:55623"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:05.940554808Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:05.980191833Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:06.036300582Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"8d5b6d85-c1bd-4824-8f1b-a9b63c674a7d","remote_addr":"[fd00:1122:3344:101::a]:55623","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":95782,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:06.036572401Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"9d5c5c8d-6325-407f-a254-e7b63578ef70","remote_addr":"[fd00:1122:3344:101::a]:55623","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":104,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:06.404546998Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:48046"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:06.40482667Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"2ed51996-2b31-4eaf-a4bd-ac180c19601b","remote_addr":"[fd00:1122:3344:101::b]:48046","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":179,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:06.438621152Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:45264"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:06.438719953Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"8d0205c5-457a-4fee-8c35-6fd9b091beec","remote_addr":"[fd00:1122:3344:101::b]:45264","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":46,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:06.453594174Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:06.453610624Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"b8f74183-f1be-4341-bc1f-08701c9d3bdb","remote_addr":"[fd00:1122:3344:101::b]:45264","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":66,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:06.467110245Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"f24abc16-224c-498a-b82f-232423f26cf5","remote_addr":"[fd00:1122:3344:101::b]:45264","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":201,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:06.576137068Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:57751"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:06.576260999Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:06.619058178Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:06.677829037Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"77c076e8-5102-47bc-874c-cec76127cca7","remote_addr":"[fd00:1122:3344:101::b]:57751","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":101617,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:06.678006938Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:42510"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:06.678167359Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"7d85fa37-d007-42dc-afdb-78445d191985","remote_addr":"[fd00:1122:3344:101::c]:42510","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":85,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:06.67830018Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"1dc04f14-bc75-4c50-b8d5-67dfa556f3f9","remote_addr":"[fd00:1122:3344:101::b]:57751","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":211,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:06.725112489Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:06.725140549Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"071cf754-bc8c-44b4-898c-b00fa1df3a56","remote_addr":"[fd00:1122:3344:101::c]:42510","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":94,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:06.793253417Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:53186"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:06.793470518Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"817c018b-649f-4051-b251-9a7b672c1b83","remote_addr":"[fd00:1122:3344:101::c]:53186","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":164,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:06.799887736Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"732b6839-799c-4736-9f46-18a1f68a4ae8","remote_addr":"[fd00:1122:3344:101::c]:42510","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":219,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:06.864946441Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:55478"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:06.865068002Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:06.90369537Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:06.961789734Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"11730c12-d890-4179-8d03-2cbd5d7a36b4","remote_addr":"[fd00:1122:3344:101::c]:55478","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":96780,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:06.962029056Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"219e6ea6-1a58-4dac-866d-68635c827c99","remote_addr":"[fd00:1122:3344:101::c]:55478","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":96,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:09.880005367Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:09.880921814Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:09.949874979Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:09.950729315Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:10.045010868Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:10.089334569Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:10.090166995Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:10.090691339Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:10.093230998Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:10.093894673Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:10.094432707Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:10.095094752Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:10.123647855Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:10.125219026Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:10.125886851Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:10.128495821Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:10.130030572Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:10.130535206Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:19.883284981Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:19.884172978Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:19.951995703Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:19.95281967Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:20.047106413Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:20.092028008Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:20.092714563Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:20.093248987Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:20.094788608Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:20.095483834Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:20.097051825Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:20.097724441Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:20.125596088Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:20.12716181Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:20.127828542Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:20.129380766Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:20.131976106Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:20.13255391Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:29.886644415Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:29.887600593Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:29.954807794Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:29.95558344Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:30.048867485Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:30.09373283Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:30.094440725Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:30.094981519Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:30.096559851Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:30.097221486Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:30.098807688Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:30.099467553Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:30.127441071Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:30.129071034Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:30.129733668Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:30.13125619Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:30.133843319Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:30.134784136Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:30.525961004Z","hostname":"sock","pid":1574,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:55034"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:30.526092715Z","hostname":"sock","pid":1574,"uri":"/baseboard","method":"GET","req_id":"be72c07e-e547-4972-8474-3dcf1d80ab19","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:55034","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":39,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:35.646422881Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:37952"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:35.646543132Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"12c286d3-53ef-442d-95ab-3c7c8882c264","remote_addr":"[fd00:1122:3344:101::a]:37952","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":53,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:35.693865425Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:35.693895265Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"382b1f25-8b79-41dc-be68-442e17ca84c3","remote_addr":"[fd00:1122:3344:101::a]:37952","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":80,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:35.702473969Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"776bef7c-67c0-41a9-9d62-48ba0ba55bf5","remote_addr":"[fd00:1122:3344:101::a]:37952","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":200,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:35.724854576Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:43953"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:35.724943487Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:35.761084136Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:35.819465022Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"46555f80-c65b-42ff-842b-c42b9e48f86e","remote_addr":"[fd00:1122:3344:101::a]:43953","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":94555,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:35.819712864Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:44806"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:35.820085637Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"ad6a68dc-de4e-49cf-91f7-a2df8295a7e7","remote_addr":"[fd00:1122:3344:101::a]:43953","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":277,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:35.820182397Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"d72e9481-7bdf-44c0-bed6-708cc78aa9e6","remote_addr":"[fd00:1122:3344:101::a]:44806","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":279,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:36.314936527Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:52991"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:36.315024038Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"25cfc10b-7075-4f52-9fec-7c3191315487","remote_addr":"[fd00:1122:3344:101::b]:52991","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":69,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:36.33541128Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:36.33544846Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"e4ae695f-cee7-4440-a35d-ffcc26a66428","remote_addr":"[fd00:1122:3344:101::b]:52991","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":69,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:36.348012104Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"99d1ce38-fc32-41bc-9b53-1abda1cd7e10","remote_addr":"[fd00:1122:3344:101::b]:52991","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":231,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:36.395211996Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:34317"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:36.395399097Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"0a3c3c16-ce78-4890-943c-8d2ed0450e89","remote_addr":"[fd00:1122:3344:101::b]:34317","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":135,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:36.433418001Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:65481"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:36.433476481Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"f5363967-b077-4549-989b-b1a93d94ba10","remote_addr":"[fd00:1122:3344:101::c]:65481","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":29,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:36.448663154Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:36.448684085Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"256025e9-bad8-4896-9e42-2ab55405ac50","remote_addr":"[fd00:1122:3344:101::c]:65481","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":38,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:36.458564668Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"8fcf63e3-4e5f-4715-9c16-efb65d5cb240","remote_addr":"[fd00:1122:3344:101::c]:65481","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":162,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:36.469380319Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:44584"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:36.469444589Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:36.50842038Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:36.566251831Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"ca4d5f30-6dee-407c-b98a-119d437c1ac9","remote_addr":"[fd00:1122:3344:101::b]:44584","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":96821,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:36.566370552Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:55992"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:36.566399653Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:49512"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:36.566616114Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:36.570838466Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"60433efc-562b-4b28-b256-5ae53d5b029a","remote_addr":"[fd00:1122:3344:101::b]:44584","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":4221,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:36.571115778Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"8085474a-7db8-479f-a752-676ea11dab26","remote_addr":"[fd00:1122:3344:101::c]:55992","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":4467,"response_code":"200"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:36.60217145Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:36.659069944Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"8dd25409-a21c-474f-a7d0-10011b42d82a","remote_addr":"[fd00:1122:3344:101::c]:49512","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":92555,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:09:36.659459757Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"18d551ab-883f-40e4-8552-92f4d5b42a71","remote_addr":"[fd00:1122:3344:101::c]:49512","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":162,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:39.89142202Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:39.893367875Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:39.957652174Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:39.95846248Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:40.051792036Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:40.095977316Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:40.096707071Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:40.097462347Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:40.097978211Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:40.098669776Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:40.101442136Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:40.102577735Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:40.129207574Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:40.130902156Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:40.131582331Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:40.133120083Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:40.135647212Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:40.137169113Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:49.894726784Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:49.89552276Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:49.961070468Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:49.961944965Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:50.054295404Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:50.098517874Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:50.099207429Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:50.099713862Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:50.100193586Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:50.100875161Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:50.103504721Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:50.10478942Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:50.131330428Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:50.13286125Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:50.133543245Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:50.135059306Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:50.136553777Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:50.139216117Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:59.898461431Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:59.899301737Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:59.963409575Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:09:59.964273812Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:00.056427219Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:00.100498838Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:00.101310984Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:00.101891918Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:00.102464692Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:00.103193258Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:00.10480126Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:00.106509083Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:00.133216822Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:00.134824804Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:00.135492809Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:00.13704453Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:00.138599252Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:00.141207921Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:00.560390398Z","hostname":"sock","pid":1574,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:57106"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:00.560537609Z","hostname":"sock","pid":1574,"uri":"/baseboard","method":"GET","req_id":"8ab0acf2-158d-43ae-888b-444a1c2473c7","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:57106","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":82,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:05.574980466Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:38036"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:05.575146217Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"b59d370c-07eb-40b6-9c9d-6bc13b1d0026","remote_addr":"[fd00:1122:3344:101::a]:38036","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":95,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:05.633367071Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:05.633400222Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"5debf4da-4c2f-4b7f-bb50-26e511901978","remote_addr":"[fd00:1122:3344:101::a]:38036","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":100,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:05.642922253Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"6842b3e2-bced-4fab-97a1-96ebbb5f7854","remote_addr":"[fd00:1122:3344:101::a]:38036","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":235,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:05.834663452Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:36224"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:05.834815014Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"632f8ad3-9e23-4008-b0bf-49d26c03c867","remote_addr":"[fd00:1122:3344:101::a]:36224","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":114,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:05.95238631Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:60912"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:05.952483091Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:05.990120222Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:06.047132707Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"2b1ec43e-de8f-456b-b4a0-1f0c2ca71644","remote_addr":"[fd00:1122:3344:101::a]:60912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":94690,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:06.04746528Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"b805e943-a6df-4ef1-8b8a-a5a67b3b8ab6","remote_addr":"[fd00:1122:3344:101::a]:60912","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":96,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:06.295217438Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:57459"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:06.295363808Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"18d95165-af80-42a5-bcd9-969e2acfce91","remote_addr":"[fd00:1122:3344:101::b]:57459","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":65,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:06.328629387Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:06.328667157Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"cb776f30-c488-469f-aa44-fbdac3cc2adc","remote_addr":"[fd00:1122:3344:101::b]:57459","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":102,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:06.338959124Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"d835750f-fec1-4db2-ab09-4db2803fbbe1","remote_addr":"[fd00:1122:3344:101::b]:57459","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":275,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:06.506559564Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:49622"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:06.506718855Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"70deee7c-6976-4dc9-8496-66a1d9b623f1","remote_addr":"[fd00:1122:3344:101::b]:49622","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":123,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:06.540564547Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:56450"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:06.540764899Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:06.583079524Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:06.640346572Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"0960b350-8877-4f25-a8d5-9c49badb9ef3","remote_addr":"[fd00:1122:3344:101::b]:56450","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":99663,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:06.640601363Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:41411"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:06.640837825Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"958a284a-213c-42e8-abc3-2b53336b7db8","remote_addr":"[fd00:1122:3344:101::b]:56450","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":245,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:06.640940056Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"c73b940b-b3f9-43f8-9495-4a7c320c544d","remote_addr":"[fd00:1122:3344:101::c]:41411","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":110,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:06.641328999Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:57105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:06.641589621Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"ffd395b6-7bb2-452c-b1e1-a47bafab4510","remote_addr":"[fd00:1122:3344:101::c]:57105","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":165,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:06.654610788Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:06.654626428Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"9886cb17-3879-4b20-a38b-f06e8a43812a","remote_addr":"[fd00:1122:3344:101::c]:41411","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":67,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:06.663070681Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"3f99f2c2-8ad0-4ff2-bf8a-8b70ed8d594c","remote_addr":"[fd00:1122:3344:101::c]:41411","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":244,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:06.774783564Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:62828"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:06.774875115Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:06.812810198Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:06.871545336Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"c50cdcd8-d20d-40e3-b273-f5b37ffd48f8","remote_addr":"[fd00:1122:3344:101::c]:62828","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":96710,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:06.871819947Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"d42ca6ed-b073-4715-8c1c-24c8cba0ee35","remote_addr":"[fd00:1122:3344:101::c]:62828","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":101,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:09.902721982Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:09.904214533Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:09.965394899Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:09.966126544Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:10.05808411Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:10.10232928Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:10.103033605Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:10.103528979Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:10.104043753Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:10.105739146Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:10.10627439Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:10.107941662Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:10.135986591Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:10.136683196Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:10.137359132Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:10.137837785Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:10.140450575Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:10.142994883Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:19.905625672Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:19.906459389Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:19.968579252Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:19.969350408Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:20.059247878Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:20.104163163Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:20.104927409Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:20.105645214Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:20.106261969Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:20.106999724Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:20.108551606Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:20.109234791Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:20.137904715Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:20.138789871Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:20.139539387Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:20.140058591Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:20.142743941Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:20.145213789Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:29.909273929Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:29.91078829Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:29.970775717Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:29.971529433Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:30.061417033Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:30.105599963Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:30.106283098Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:30.10784436Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:30.108389624Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:30.109023919Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:30.1105584Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:30.111190985Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:30.139788568Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:30.140318972Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:30.141993934Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:30.142478118Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:30.145090677Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:30.146626639Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:30.593636713Z","hostname":"sock","pid":1574,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:63832"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:30.593821554Z","hostname":"sock","pid":1574,"uri":"/baseboard","method":"GET","req_id":"bab5fbca-9090-49ff-866c-264626232cdf","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:63832","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":76,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:35.608471183Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:40823"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:35.608606464Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"f0177c79-e146-4ca6-b2ed-6cd565814bfa","remote_addr":"[fd00:1122:3344:101::a]:40823","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":62,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:35.651115551Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:35.651145301Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"19f81e49-092e-4526-bb7b-d419f91ef97e","remote_addr":"[fd00:1122:3344:101::a]:40823","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":66,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:35.659950617Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"6a15246a-09be-4e93-9def-c4c5f5f5532c","remote_addr":"[fd00:1122:3344:101::a]:40823","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":186,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:35.689630058Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:44327"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:35.689707009Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:35.729837058Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:35.789424082Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"24ac66c1-5141-4bb9-9295-78ec00ee73dc","remote_addr":"[fd00:1122:3344:101::a]:44327","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":99750,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:35.789525933Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:58765"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:35.789847266Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"da60e905-b1f7-48ac-9009-dca3ab45547e","remote_addr":"[fd00:1122:3344:101::a]:58765","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":229,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:35.789914716Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"63d3f5ca-06b5-4e11-af04-13a9666ff104","remote_addr":"[fd00:1122:3344:101::a]:44327","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":222,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:36.306281567Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:46641"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:36.306380558Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"5c505c68-2c5a-4003-8e85-3db96f3684f2","remote_addr":"[fd00:1122:3344:101::b]:46641","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":52,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:36.363815646Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:36.363850787Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"5d3f9470-5c56-45de-b939-dd4031c24953","remote_addr":"[fd00:1122:3344:101::b]:46641","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":68,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:36.36564441Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:56742"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:36.365719861Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"e5977b7b-e91f-48c0-a25b-7c2ace11e0c0","remote_addr":"[fd00:1122:3344:101::c]:56742","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":30,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:36.423231129Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:36.42329083Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"d9c1f286-2294-49bd-9aea-6ba6ac6fc1d2","remote_addr":"[fd00:1122:3344:101::c]:56742","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":125,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:36.432022285Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"f6117cff-10fe-4b8c-8201-924ef1a5c35a","remote_addr":"[fd00:1122:3344:101::c]:56742","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":188,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:36.490946455Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:57724"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:36.491100556Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"aa012cb8-ebd3-4a17-893d-c54579ac8c56","remote_addr":"[fd00:1122:3344:101::b]:57724","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":94,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:36.499896821Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"77d742b9-ab10-44d5-9a89-38c52589d9a7","remote_addr":"[fd00:1122:3344:101::b]:46641","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":115,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:36.507298406Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:48207"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:36.507370407Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:36.54805827Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:36.611169301Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"f7cd15ea-457b-41e9-9ad6-fa697d6855ea","remote_addr":"[fd00:1122:3344:101::c]:48207","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":103814,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:36.611286602Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:41589"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:36.611335952Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:53498"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:36.611578814Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"87dfa3cd-d6e9-4746-9609-3eb5ac597567","remote_addr":"[fd00:1122:3344:101::c]:53498","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":173,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:36.611665055Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"0f0cb230-b943-4337-9752-09a424ee9049","remote_addr":"[fd00:1122:3344:101::c]:48207","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":230,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:36.611720515Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:36.652005066Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:36.709700606Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"212ca1eb-9463-4490-bb8e-c40d05424ebb","remote_addr":"[fd00:1122:3344:101::b]:41589","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":98109,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:10:36.710076249Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"6ce7101c-b25a-492f-adf8-f4029457e70b","remote_addr":"[fd00:1122:3344:101::b]:41589","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":150,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:39.912409872Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:39.914156094Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:39.97395031Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:39.974789657Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:40.064144633Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:40.108107051Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:40.10930707Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:40.109856954Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:40.110389638Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:40.111077413Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:40.112614194Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:40.11330367Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:40.141852713Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:40.142369336Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:40.144070619Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:40.144552063Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:40.147090602Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:40.148659594Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:49.917075345Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:49.918767198Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:49.976407928Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:49.977270084Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:50.066358748Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:50.110345137Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:50.111145882Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:50.111729117Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:50.112251701Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:50.112910416Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:50.114460997Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:50.115106632Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:50.143688195Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:50.14424432Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:50.145925852Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:50.146423066Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:50.149068656Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:50.151135711Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:59.92850396Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:59.929370936Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:59.978074889Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:10:59.978921066Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:00.06801855Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:00.111982988Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:00.112774484Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:00.113317468Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:00.113866712Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:00.114550107Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:00.116112849Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:00.116774984Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:00.145348157Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:00.145927191Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:00.147604644Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:00.148093037Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:00.150725417Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:00.153357607Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:00.626134713Z","hostname":"sock","pid":1574,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:33333"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:00.626237464Z","hostname":"sock","pid":1574,"uri":"/baseboard","method":"GET","req_id":"d1e7626d-1144-4906-9ff7-0b236b930b4e","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:33333","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":30,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:05.748976808Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:59465"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:05.749101339Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"5d03606d-095b-47eb-8b2e-1ef0bd042ca3","remote_addr":"[fd00:1122:3344:101::a]:59465","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":55,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:05.795997959Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:05.796031499Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"f05846c7-5f7f-4f96-b4ff-18a14a591f02","remote_addr":"[fd00:1122:3344:101::a]:59465","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":77,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:05.805122147Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"4cd84620-e3a1-4e63-ad3d-8ef007c0695a","remote_addr":"[fd00:1122:3344:101::a]:59465","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":204,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:05.991656858Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:37846"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:05.991827159Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"995049b0-74d3-4359-8509-1c0b1cc75946","remote_addr":"[fd00:1122:3344:101::a]:37846","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":134,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.023367965Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:41990"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.023466575Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.023486286Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.023531316Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.025137448Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.039427005Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.039494565Z","hostname":"sock","pid":1574,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_473f0b4e-a714-496d-8d5d-665782986893"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.070206404Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.070342405Z","hostname":"sock","pid":1574,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.07639626Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:41287"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.076720423Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.104870823Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.104921423Z","hostname":"sock","pid":1574,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.13003514Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.153765707Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.153871828Z","hostname":"sock","pid":1574,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_2536a198-fc1d-4175-aab5-a62f1506f3e4"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.198049838Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.198158718Z","hostname":"sock","pid":1574,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_dd78d2d6-0d80-45fe-9f73-e75ef7da86ee"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.24131993Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.241392791Z","hostname":"sock","pid":1574,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_28b058a3-a4b5-473e-8580-f5c606b6d30b"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.248638345Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:36069"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.249485521Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3b3dea8a-e84f-4acf-bafc-3b0e4a815dc5, kind: Service { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 }, name: Name(\"external-dns-be35e232-dd9b-4d59-b4e1-eadb7f69bb77\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 109])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 95dce781-7d6b-4bf1-88f1-880b60d0adba, kind: Service { id: 14e637a2-51ce-4cac-b831-85b757e5c55d }, name: Name(\"external-dns-14e637a2-51ce-4cac-b831-85b757e5c55d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 238])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 5dbd0a76-35d9-4b29-8800-824968d1caba, kind: Service { id: 9b2d2174-b236-4546-bace-300e05afe3c2 }, name: Name(\"nexus-9b2d2174-b236-4546-bace-300e05afe3c2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 121])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: db0c0ea5-0d8f-4f9d-9979-f3989ce962ea, kind: Service { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 }, name: Name(\"nexus-16a7178a-dbb6-4392-b395-7bddb3817d13\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e67bea26-1a67-4166-875f-3cab431644e6, kind: Service { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 }, name: Name(\"nexus-3dce42ed-74cb-4c8b-9326-8273a1efd563\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 239])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.249613282Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.249631912Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"544381db-50b6-44ab-81fa-cc2b47bc8e25","remote_addr":"[fd00:1122:3344:101::a]:41287","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":173037,"response_code":"204"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.249707633Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.249740093Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.249756723Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.249802604Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.249850094Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.249873234Z","hostname":"sock","pid":1574,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"e00536dd-f171-4958-b1d8-cc97433b6c69","remote_addr":"[fd00:1122:3344:101::a]:36069","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":866,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.250083606Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"bf6206a5-8b66-4f45-b6e4-0b8f09b46ca0","remote_addr":"[fd00:1122:3344:101::a]:41287","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":191,"response_code":"200"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.293459509Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.29354039Z","hostname":"sock","pid":1574,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_80f7cc50-8d94-4fb0-b296-446e3acf3366"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.342419154Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.342511975Z","hostname":"sock","pid":1574,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_feac0a3f-e4dc-4a16-9eeb-839f74ad9665"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.384422188Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.384505048Z","hostname":"sock","pid":1574,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_f334d74e-134c-4206-a5ba-9a6ac6ea7954"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.408367346Z","hostname":"sock","pid":1574,"uri":"/inventory","method":"GET","req_id":"e0f82c87-16d5-4943-af14-4abc51d86a8c","remote_addr":"[fd00:1122:3344:101::a]:41990","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":384887,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.472525525Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:43674"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.472624636Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"f8e92332-81f0-41d7-8d82-ee38f80d1cf4","remote_addr":"[fd00:1122:3344:101::b]:43674","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":43,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.52814923Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.52818109Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"26c973ef-0a66-4340-ba18-dbe05f1ad9af","remote_addr":"[fd00:1122:3344:101::b]:43674","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":76,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.540508792Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"708dc693-c3d1-425f-a6b4-11c463218977","remote_addr":"[fd00:1122:3344:101::b]:43674","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":263,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.660411746Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:45269"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.660528897Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"584772b6-ce13-4cac-95bf-b561ebef156e","remote_addr":"[fd00:1122:3344:101::c]:45269","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":52,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.686917874Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:59126"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.687032655Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.687041995Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.687095755Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.687375557Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.703014754Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.703068414Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"0a6200ea-12d0-4f96-be35-8ca1e66f9eb3","remote_addr":"[fd00:1122:3344:101::c]:45269","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":207,"response_code":"204"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.703384177Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.703401017Z","hostname":"sock","pid":1574,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_473f0b4e-a714-496d-8d5d-665782986893"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.709541903Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:56384"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.709931675Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"3dd804e9-df05-42b6-b4da-32697b872fbd","remote_addr":"[fd00:1122:3344:101::c]:56384","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":255,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.711392846Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"55313518-365c-452b-9424-ea95c065e092","remote_addr":"[fd00:1122:3344:101::c]:45269","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":345,"response_code":"204"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.734209326Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.734268906Z","hostname":"sock","pid":1574,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.760492992Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.760545972Z","hostname":"sock","pid":1574,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.783265302Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.783331802Z","hostname":"sock","pid":1574,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_2536a198-fc1d-4175-aab5-a62f1506f3e4"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.783340502Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:34196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.786715978Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"7bbedc6c-edd4-4bb6-b57c-2b7f3b000125","remote_addr":"[fd00:1122:3344:101::b]:34196","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":259,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.789419528Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:61563"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.792265269Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:32781"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.792474081Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3b3dea8a-e84f-4acf-bafc-3b0e4a815dc5, kind: Service { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 }, name: Name(\"external-dns-be35e232-dd9b-4d59-b4e1-eadb7f69bb77\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 109])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 95dce781-7d6b-4bf1-88f1-880b60d0adba, kind: Service { id: 14e637a2-51ce-4cac-b831-85b757e5c55d }, name: Name(\"external-dns-14e637a2-51ce-4cac-b831-85b757e5c55d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 238])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 5dbd0a76-35d9-4b29-8800-824968d1caba, kind: Service { id: 9b2d2174-b236-4546-bace-300e05afe3c2 }, name: Name(\"nexus-9b2d2174-b236-4546-bace-300e05afe3c2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 121])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: db0c0ea5-0d8f-4f9d-9979-f3989ce962ea, kind: Service { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 }, name: Name(\"nexus-16a7178a-dbb6-4392-b395-7bddb3817d13\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e67bea26-1a67-4166-875f-3cab431644e6, kind: Service { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 }, name: Name(\"nexus-3dce42ed-74cb-4c8b-9326-8273a1efd563\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 239])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.792507111Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.792543331Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.792554081Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.792559821Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.792581131Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.792593371Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.792610662Z","hostname":"sock","pid":1574,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"1d22d2c0-dab9-4033-ae1e-fe958aa5fadd","remote_addr":"[fd00:1122:3344:101::b]:32781","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":286,"response_code":"204"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.794834988Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.794844298Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Reading ledger from /pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.794934309Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Reading ledger from /pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.80177909Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"common/src/ledger.rs:178"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.80983706Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.80989066Z","hostname":"sock","pid":1574,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_dd78d2d6-0d80-45fe-9f73-e75ef7da86ee"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.817034164Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:46608"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.817413176Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 473f0b4e-a714-496d-8d5d-665782986893 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.826774746Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.826825867Z","hostname":"sock","pid":1574,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_473f0b4e-a714-496d-8d5d-665782986893"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.842285402Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:44093"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.858772875Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.859231708Z","hostname":"sock","pid":1574,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.864099385Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.875819002Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.875864183Z","hostname":"sock","pid":1574,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_28b058a3-a4b5-473e-8580-f5c606b6d30b"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.89433358Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.894543582Z","hostname":"sock","pid":1574,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.913959696Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.914009657Z","hostname":"sock","pid":1574,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_80f7cc50-8d94-4fb0-b296-446e3acf3366"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 2536a198-fc1d-4175-aab5-a62f1506f3e4 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.941715083Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.942119966Z","hostname":"sock","pid":1574,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_2536a198-fc1d-4175-aab5-a62f1506f3e4"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.947635328Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:64662"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.948147542Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:731","rules":"[ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3b3dea8a-e84f-4acf-bafc-3b0e4a815dc5, kind: Service { id: be35e232-dd9b-4d59-b4e1-eadb7f69bb77 }, name: Name(\"external-dns-be35e232-dd9b-4d59-b4e1-eadb7f69bb77\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 185, 109])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 95dce781-7d6b-4bf1-88f1-880b60d0adba, kind: Service { id: 14e637a2-51ce-4cac-b831-85b757e5c55d }, name: Name(\"external-dns-14e637a2-51ce-4cac-b831-85b757e5c55d\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 238])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 5dbd0a76-35d9-4b29-8800-824968d1caba, kind: Service { id: 9b2d2174-b236-4546-bace-300e05afe3c2 }, name: Name(\"nexus-9b2d2174-b236-4546-bace-300e05afe3c2\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 201, 121])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: db0c0ea5-0d8f-4f9d-9979-f3989ce962ea, kind: Service { id: 16a7178a-dbb6-4392-b395-7bddb3817d13 }, name: Name(\"nexus-16a7178a-dbb6-4392-b395-7bddb3817d13\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 242, 19])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: e67bea26-1a67-4166-875f-3cab431644e6, kind: Service { id: 3dce42ed-74cb-4c8b-9326-8273a1efd563 }, name: Name(\"nexus-3dce42ed-74cb-4c8b-9326-8273a1efd563\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 239])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.948197622Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.948246752Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.948270363Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.948283123Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.948323383Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.948345853Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:750","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.948374453Z","hostname":"sock","pid":1574,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"fa4c90f7-a6a7-4b02-8e04-ebc76ee267c8","remote_addr":"[fd00:1122:3344:101::c]:64662","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":545,"response_code":"204"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.952048741Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.952061221Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"df6ab9f0-b505-4d83-b428-397bc36b2bdb","remote_addr":"[fd00:1122:3344:101::c]:46608","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":134857,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.957080868Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"aab9b241-ef6a-4046-adb5-6625f0e7b267","remote_addr":"[fd00:1122:3344:101::c]:46608","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":785,"response_code":"200"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.957890584Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.957905874Z","hostname":"sock","pid":1574,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_feac0a3f-e4dc-4a16-9eeb-839f74ad9665"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.985283459Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.985679902Z","hostname":"sock","pid":1574,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_f334d74e-134c-4206-a5ba-9a6ac6ea7954"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:06.995119042Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: dd78d2d6-0d80-45fe-9f73-e75ef7da86ee (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:07.000771134Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:07.000858695Z","hostname":"sock","pid":1574,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_dd78d2d6-0d80-45fe-9f73-e75ef7da86ee"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:07.00830932Z","hostname":"sock","pid":1574,"uri":"/inventory","method":"GET","req_id":"bc82d523-545a-4652-bbcf-2ea57afbd9b7","remote_addr":"[fd00:1122:3344:101::b]:59126","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":321294,"response_code":"200"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 28b058a3-a4b5-473e-8580-f5c606b6d30b (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:07.033996822Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:07.034048562Z","hostname":"sock","pid":1574,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_28b058a3-a4b5-473e-8580-f5c606b6d30b"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: 80f7cc50-8d94-4fb0-b296-446e3acf3366 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:07.064356648Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:07.064412279Z","hostname":"sock","pid":1574,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_80f7cc50-8d94-4fb0-b296-446e3acf3366"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:07.070368103Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"4aa468fb-2478-4327-b5b4-2349d77afacc","remote_addr":"[fd00:1122:3344:101::b]:44093","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":227891,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:07.070949858Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"791e284f-4435-4521-a8b1-9572f8823169","remote_addr":"[fd00:1122:3344:101::b]:44093","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":284,"response_code":"200"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: feac0a3f-e4dc-4a16-9eeb-839f74ad9665 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:07.094707355Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:07.094940416Z","hostname":"sock","pid":1574,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_feac0a3f-e4dc-4a16-9eeb-839f74ad9665"} {"msg":"Received DatasetsList { zpool: ZpoolName { id: f334d74e-134c-4206-a5ba-9a6ac6ea7954 (zpool), kind: External }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:07.122938285Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Listing datasets within zpool","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:07.122986666Z","hostname":"sock","pid":1574,"request":"datasets_list","component":"StorageManager","file":"sled-storage/src/manager.rs:1022","zpool":"oxp_f334d74e-134c-4206-a5ba-9a6ac6ea7954"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:07.136489176Z","hostname":"sock","pid":1574,"uri":"/inventory","method":"GET","req_id":"7a4b28fa-6b08-4ce8-b067-a63590149c4b","remote_addr":"[fd00:1122:3344:101::c]:61563","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":346976,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:09.931100378Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:09.931884834Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:09.980087994Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:09.980766628Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:10.070023514Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:10.113930152Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:10.114634567Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:10.115186441Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:10.115665265Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:10.116298259Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:10.117799721Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:10.118460655Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:10.147083019Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:10.147661693Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:10.149357936Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:10.150952518Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:10.151504132Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:10.155165789Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:19.935403059Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:19.936212826Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:19.982071428Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:19.982758093Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:20.073040566Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:20.115961906Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:20.116648211Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:20.117185025Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:20.117668759Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:20.118336384Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:20.119896535Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:20.12055922Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:20.149206704Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:20.149867519Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:20.151747213Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:20.153434915Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:20.15401732Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:20.157974999Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:29.939116846Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:29.939861092Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:29.984735227Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:29.985474652Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:30.075524424Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:30.118369413Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:30.119058198Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:30.119601492Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:30.120107106Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:30.120765701Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:30.122303373Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:30.122939557Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:30.151554101Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:30.152155415Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:30.15280839Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:30.155339109Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:30.155830503Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:30.159699421Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:30.658452611Z","hostname":"sock","pid":1574,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:50776"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:30.658577872Z","hostname":"sock","pid":1574,"uri":"/baseboard","method":"GET","req_id":"b508ca9b-3d57-42f5-99c6-fd7df7f1cb39","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:50776","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":85,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:35.617986159Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:46418"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:35.61810356Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"48bf429d-ab34-4810-85ea-e5cb210cf418","remote_addr":"[fd00:1122:3344:101::a]:46418","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":46,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:35.660101173Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:35.660116063Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"2ee8daae-a4b0-4bf4-a1ca-d3092c0710e1","remote_addr":"[fd00:1122:3344:101::a]:46418","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":57,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:35.667328817Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:59491"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:35.667498598Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"34254a3d-9508-4c61-a021-d1eff35a8b05","remote_addr":"[fd00:1122:3344:101::a]:59491","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":98,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:35.669758855Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"58d1974d-9c30-4a60-ba66-490f6c43f3ee","remote_addr":"[fd00:1122:3344:101::a]:46418","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":147,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:35.684226053Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:60463"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:35.684313214Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:35.724800776Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:35.784549711Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"4ecd8202-fd02-4869-9cd8-bf765fc7c6a3","remote_addr":"[fd00:1122:3344:101::a]:60463","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":100261,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:35.784896194Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"6074375b-361b-4026-8e7f-63e2a861ecd9","remote_addr":"[fd00:1122:3344:101::a]:60463","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":107,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:36.288738311Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:59048"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:36.288871452Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"951b5f6e-5035-409b-8c07-dbc0f964ff7c","remote_addr":"[fd00:1122:3344:101::b]:59048","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":60,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:36.353243283Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:36.353293503Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"cc83609d-668b-4322-9a17-a82cfa9b0d45","remote_addr":"[fd00:1122:3344:101::b]:59048","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":75,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:36.362374601Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"8ac0a4c0-7abc-442c-a835-412452ae10a2","remote_addr":"[fd00:1122:3344:101::b]:59048","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":251,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:36.366054008Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:65078"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:36.366226849Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"1ac43fc6-8af4-4bb3-ac84-b977bf63a89d","remote_addr":"[fd00:1122:3344:101::b]:65078","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":110,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:36.436633714Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:43697"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:36.436728505Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:36.476659113Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:36.542804836Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"90b53eb8-d35a-40f6-a6d0-f04c79cb4966","remote_addr":"[fd00:1122:3344:101::b]:43697","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":106120,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:36.543041878Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:51010"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:36.543116029Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:36768"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:36.543367881Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"83e2747d-c77d-4a8f-a33e-7e4b459ac55c","remote_addr":"[fd00:1122:3344:101::b]:43697","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":292,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:36.543460441Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:36.543477421Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"7c72a7a8-0c1a-47a1-8e15-b47cb3d642ac","remote_addr":"[fd00:1122:3344:101::c]:51010","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":135,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:36.564183216Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:36.564259416Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"d5345d93-4c94-4e53-b5b9-faf6960142f8","remote_addr":"[fd00:1122:3344:101::c]:51010","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":205,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:36.576254576Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"976c9490-e7c1-48bb-acc3-469e6fc84c07","remote_addr":"[fd00:1122:3344:101::c]:51010","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":865,"response_code":"204"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:36.584364766Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:36.623684929Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:44672"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:36.623923301Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"28593601-f400-41b8-a271-e73c24d461a7","remote_addr":"[fd00:1122:3344:101::c]:44672","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":145,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:36.640639646Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"8192d34e-d051-4606-820c-d4f5f83cc5c2","remote_addr":"[fd00:1122:3344:101::c]:36768","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":97322,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:11:36.640932318Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"ce598d12-3a3a-41aa-a628-ac78e925cead","remote_addr":"[fd00:1122:3344:101::c]:36768","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":136,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:39.943250376Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:39.943957371Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:39.986951572Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:39.987750708Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:40.078062262Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:40.121231113Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:40.121973409Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:40.122521433Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:40.123036737Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:40.123711742Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:40.124228506Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:40.124874731Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:40.153907797Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:40.154495412Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:40.155146577Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:40.158208199Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:40.158777554Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:40.161348483Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:49.948007071Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:49.948774467Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:49.989027597Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:49.989800093Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:50.079607642Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:50.127955033Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:50.128667228Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:50.129324913Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:50.129847147Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:50.130520922Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:50.131165317Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:50.131814242Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:50.15565248Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:50.156388065Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:50.15702795Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:50.160121133Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:50.160674847Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:50.163231056Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:59.951280214Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:59.95203865Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:59.99096061Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:11:59.991669596Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:00.081891818Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:00.129751505Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:00.13043476Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:00.130928364Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:00.131395468Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:00.132049563Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:00.132530466Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:00.133198641Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:00.157824315Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:00.158347449Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:00.158988644Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:00.161598853Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:00.162186097Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:00.165135619Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:00.69223045Z","hostname":"sock","pid":1574,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:60825"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:00.692394882Z","hostname":"sock","pid":1574,"uri":"/baseboard","method":"GET","req_id":"e324201c-b8d3-4249-82c4-a808fbf8be92","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:60825","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":59,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:05.733374117Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:42626"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:05.733534788Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"4aef012f-5446-482b-a402-cbeba72d29d8","remote_addr":"[fd00:1122:3344:101::a]:42626","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":97,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:05.785401355Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:58593"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:05.785607226Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"322ebe38-f526-4302-8714-02ba8eb774e8","remote_addr":"[fd00:1122:3344:101::a]:58593","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":123,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:05.828004102Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:05.828038883Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"34a344a9-feef-4cab-8be7-9c1661cc2a33","remote_addr":"[fd00:1122:3344:101::a]:42626","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":73,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:05.83700341Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"4e9ee115-c92c-4eef-bfc5-ab3a9b0d9ee4","remote_addr":"[fd00:1122:3344:101::a]:42626","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":205,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:06.062163829Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:44577"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:06.06225487Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:06.100048301Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:06.193207076Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"5cd9f879-ee01-482c-a045-bc16ecc45e32","remote_addr":"[fd00:1122:3344:101::a]:44577","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":130979,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:06.19372961Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"da7a16ef-4528-4725-88df-2ff3a43bab91","remote_addr":"[fd00:1122:3344:101::a]:44577","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":213,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:06.417736021Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:44289"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:06.417831551Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"2cbbec0d-7cbb-4789-806c-39ca8966bcba","remote_addr":"[fd00:1122:3344:101::b]:44289","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":55,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:06.459319101Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:06.459340391Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"1aac9697-fe01-4661-98b2-80e5099d3541","remote_addr":"[fd00:1122:3344:101::b]:44289","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":59,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:06.468711731Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"5982389d-21cb-4173-b451-cc82d29aa02b","remote_addr":"[fd00:1122:3344:101::b]:44289","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":200,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:06.556838868Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:62338"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:06.556987159Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"863e59fa-9570-4031-b69f-ce6c435aebe9","remote_addr":"[fd00:1122:3344:101::b]:62338","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":119,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:06.571367547Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:53006"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:06.571529288Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"0b6af2ae-a467-4a36-b0a3-54d4828de6de","remote_addr":"[fd00:1122:3344:101::c]:53006","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":84,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:06.601210759Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:34826"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:06.60135652Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:06.640842925Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:06.698532375Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"a3c7580a-6c10-4ad7-bd74-af34203e0bc4","remote_addr":"[fd00:1122:3344:101::b]:34826","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":97249,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:06.698721466Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:56979"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:06.698825177Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"bbaae151-2ab1-4c27-b0a5-1e991647812e","remote_addr":"[fd00:1122:3344:101::b]:34826","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":131,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:06.698890228Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"a45ce921-3e2c-4444-9ee3-1aa98486c4c4","remote_addr":"[fd00:1122:3344:101::c]:56979","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":83,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:06.713271775Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:06.713288285Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"a0b17ac3-0281-4562-baa6-1b576aae393b","remote_addr":"[fd00:1122:3344:101::c]:56979","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":98,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:06.721883419Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"4969e141-720c-44fd-87bb-537ea453c683","remote_addr":"[fd00:1122:3344:101::c]:56979","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":258,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:06.749268613Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:54144"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:06.749374004Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:06.788951049Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:06.84402471Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"53dce4ea-b65d-45a3-b5d3-d44df178e8d4","remote_addr":"[fd00:1122:3344:101::c]:54144","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":94693,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:06.844358082Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"4fd20bb2-b05b-48b1-a80b-2a15d56ab5b5","remote_addr":"[fd00:1122:3344:101::c]:54144","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":112,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:09.955194663Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:09.955957358Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:09.992853753Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:09.993653279Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:10.083845992Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:10.131705149Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:10.132389234Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:10.132908068Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:10.133415632Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:10.134080097Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:10.134615891Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:10.135265036Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:10.160221692Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:10.161026018Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:10.161694393Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:10.163193014Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:10.163700978Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:10.166749721Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:19.959136821Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:19.959886347Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:19.994687886Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:19.995404151Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:20.088040322Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:20.134927372Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:20.135624147Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:20.136160571Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:20.136681755Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:20.13735843Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:20.137881914Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:20.138532029Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:20.162210836Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:20.162742112Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:20.163370392Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:20.164933146Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:20.16542409Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:20.169006466Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:23.31515696Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:45937"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:23.315433292Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"3743e9da-a926-4055-9920-bc186488dfb4","remote_addr":"[fd00:1122:3344:101::a]:45937","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":211,"response_code":"200"} {"msg":"Mapping virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:23.315689754Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:838","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(15890920) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:23.315710564Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"PUT","req_id":"8a68075b-fe05-4b58-a10b-126e65518508","remote_addr":"[fd00:1122:3344:101::a]:45937","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":98,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:23.366414912Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:33649"} {"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:23.373046582Z","hostname":"sock","pid":1574,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:555","metadata":"InstanceMetadata { silo_id: ea8b3055-5ecf-4ed0-93a8-fdadd18e20b8, project_id: 53281266-8693-4b0d-8c79-19e20f52ead1 }","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-11-18T21:12:23.366218871Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 22b961b8-9080-4e1f-8bc7-6427df717a9f, kind: Instance { id: 229c467e-2b02-41ee-af58-c382cbfa394d }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(15890920), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 22b961b8-9080-4e1f-8bc7-6427df717a9f, kind: Instance { id: 229c467e-2b02-41ee-af58-c382cbfa394d }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(15890920), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 22b961b8-9080-4e1f-8bc7-6427df717a9f, kind: Instance { id: 229c467e-2b02-41ee-af58-c382cbfa394d }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(15890920), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(15890920))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 22b961b8-9080-4e1f-8bc7-6427df717a9f, kind: Instance { id: 229c467e-2b02-41ee-af58-c382cbfa394d }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(15890920), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-fad5d4a7e70\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 405d5a6d-ad9c-446e-b031-e37679ff4fe9, read_only_parent: Some(Volume { block_size: 512, id: bffb438a-208b-4066-b218-cea9591433fa, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: d05ec865-8dd6-40fc-84a5-955347d159c5, key: Some(\"wkBXd1fH13UhURs2XmC84mLujXfEY2p2FIjX5+z8jX0=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::16]:19001\", \"[fd00:1122:3344:101::15]:19002\", \"[fd00:1122:3344:101::14]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 405d5a6d-ad9c-446e-b031-e37679ff4fe9, key: Some(\"P2bfpgh9p0fbT/LCwO4vjG9B9Aa0FjCGGdTke6BpKcs=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::17]:19001\", \"[fd00:1122:3344:101::16]:19002\", \"[fd00:1122:3344:101::14]:19002\"] } }] } }], boot_settings: Some(BootSettings { order: [BootOrderEntry { name: \"disk-fad5d4a7e70\" }] }), cloud_init_bytes: Some(..) }","migration_id":"None","propolis_id":"b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d"} {"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:23.373093982Z","hostname":"sock","pid":1574,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:588","migration_id":"None","propolis_id":"b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d"} {"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:23.373114602Z","hostname":"sock","pid":1574,"propolis_id":"b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","component":"InstanceManager","file":"sled-agent/src/instance.rs:1281","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 22b961b8-9080-4e1f-8bc7-6427df717a9f, kind: Instance { id: 229c467e-2b02-41ee-af58-c382cbfa394d }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(15890920), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 22b961b8-9080-4e1f-8bc7-6427df717a9f, kind: Instance { id: 229c467e-2b02-41ee-af58-c382cbfa394d }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(15890920), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 22b961b8-9080-4e1f-8bc7-6427df717a9f, kind: Instance { id: 229c467e-2b02-41ee-af58-c382cbfa394d }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(15890920), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(15890920))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, ResolvedVpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 22b961b8-9080-4e1f-8bc7-6427df717a9f, kind: Instance { id: 229c467e-2b02-41ee-af58-c382cbfa394d }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 240, 0, 0])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(15890920), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-fad5d4a7e70\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 405d5a6d-ad9c-446e-b031-e37679ff4fe9, read_only_parent: Some(Volume { block_size: 512, id: bffb438a-208b-4066-b218-cea9591433fa, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: d05ec865-8dd6-40fc-84a5-955347d159c5, key: Some(\"wkBXd1fH13UhURs2XmC84mLujXfEY2p2FIjX5+z8jX0=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::16]:19001\", \"[fd00:1122:3344:101::15]:19002\", \"[fd00:1122:3344:101::14]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 405d5a6d-ad9c-446e-b031-e37679ff4fe9, key: Some(\"P2bfpgh9p0fbT/LCwO4vjG9B9Aa0FjCGGdTke6BpKcs=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::17]:19001\", \"[fd00:1122:3344:101::16]:19002\", \"[fd00:1122:3344:101::14]:19002\"] } }] } }], boot_settings: Some(BootSettings { order: [BootOrderEntry { name: \"disk-fad5d4a7e70\" }] }), cloud_init_bytes: Some(..) }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-11-18T21:12:23.366218871Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400, migration_id: None }","migration_id":"None","propolis_id":"b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:23.373218633Z","hostname":"sock","pid":1574,"uri":"/vmms/b0157dea-ec31-4457-96e1-5fc35491c963","method":"PUT","req_id":"1b83ec68-00e7-44a3-85dd-0d73b05b666a","remote_addr":"[fd00:1122:3344:101::a]:33649","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":6769,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:23.451608348Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:41897"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:23.452736266Z","hostname":"sock","pid":1574,"uri":"/vmms/b0157dea-ec31-4457-96e1-5fc35491c963/state","method":"PUT","req_id":"a694552b-d1fd-45db-bcc8-d7848dcde115","remote_addr":"[fd00:1122:3344:101::a]:41897","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":1027,"response_code":"200"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:23.457900195Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:430","port":"Port { inner: PortInner(PortData { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 240, 0, 0]), slot: 0, vni: Vni { inner: 15890920 }, subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 } }) }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:23.457942885Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"Configuring new Omicron zone: oxz_propolis-server_b0157dea-ec31-4457-96e1-5fc35491c963","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:23.500897126Z","hostname":"sock","pid":1574,"instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","propolis_id":"b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","component":"InstanceManager","file":"illumos-utils/src/zone.rs:325"} {"msg":"Installing Omicron zone: oxz_propolis-server_b0157dea-ec31-4457-96e1-5fc35491c963","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:23.522371566Z","hostname":"sock","pid":1574,"instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","propolis_id":"b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","component":"InstanceManager","file":"illumos-utils/src/zone.rs:363"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:23.537323927Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:60911"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:23.537504259Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"d559a616-392f-4620-9183-b9b69e69c1de","remote_addr":"[fd00:1122:3344:101::a]:60911","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":86,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:23.554643646Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {22b961b8-9080-4e1f-8bc7-6427df717a9f: {10.151.2.176: {3ac43a21-5b43-44f3-aa9c-4f7a4e11405c}, 10.151.2.175: {3ac43a21-5b43-44f3-aa9c-4f7a4e11405c}}} }"} {"msg":"Profile for oxz_propolis-server_b0157dea-ec31-4457-96e1-5fc35491c963:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:24.962208684Z","hostname":"sock","pid":1574,"instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","propolis_id":"b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_propolis-server_b0157dea-ec31-4457-96e1-5fc35491c963 zone","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:24.962388375Z","hostname":"sock","pid":1574,"zone":"oxz_propolis-server_b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","propolis_id":"b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:488"} {"msg":"wait for service Some(\"oxz_propolis-server_b0157dea-ec31-4457-96e1-5fc35491c963\") failed: Property not found. retry in 44.130865ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:25.163377874Z","hostname":"sock","pid":1574,"zone":"oxz_propolis-server_b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","propolis_id":"b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_b0157dea-ec31-4457-96e1-5fc35491c963\") failed: Property not found. retry in 109.420845ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:25.221474687Z","hostname":"sock","pid":1574,"zone":"oxz_propolis-server_b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","propolis_id":"b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_b0157dea-ec31-4457-96e1-5fc35491c963\") failed: Property not found. retry in 176.273091ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:25.349813615Z","hostname":"sock","pid":1574,"zone":"oxz_propolis-server_b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","propolis_id":"b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_b0157dea-ec31-4457-96e1-5fc35491c963\") failed: Property not found. retry in 405.402463ms","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:25.541293232Z","hostname":"sock","pid":1574,"zone":"oxz_propolis-server_b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","propolis_id":"b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_b0157dea-ec31-4457-96e1-5fc35491c963\") failed: Property not found. retry in 1.059335263s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:25.965187264Z","hostname":"sock","pid":1574,"zone":"oxz_propolis-server_b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","propolis_id":"b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_b0157dea-ec31-4457-96e1-5fc35491c963\") failed: Property not found. retry in 1.207995969s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:27.039126723Z","hostname":"sock","pid":1574,"zone":"oxz_propolis-server_b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","propolis_id":"b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_b0157dea-ec31-4457-96e1-5fc35491c963\") failed: Property not found. retry in 1.307625036s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:28.262057864Z","hostname":"sock","pid":1574,"zone":"oxz_propolis-server_b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","propolis_id":"b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_b0157dea-ec31-4457-96e1-5fc35491c963\") failed: Property not found. retry in 1.228517614s","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:29.583883592Z","hostname":"sock","pid":1574,"zone":"oxz_propolis-server_b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","propolis_id":"b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:29.962891288Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:29.964362779Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:29.99801144Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:29.998693466Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:30.090335959Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:30.138143755Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:30.138815671Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:30.139416695Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:30.140779795Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:30.14143748Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:30.14273456Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:30.143379945Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:30.165064626Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:30.165859942Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:30.166569427Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:30.167270093Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:30.167956788Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:30.171201792Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:30.727169688Z","hostname":"sock","pid":1574,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:63214"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:30.727341479Z","hostname":"sock","pid":1574,"uri":"/baseboard","method":"GET","req_id":"adfd62dc-5751-4fbf-a536-a51d209af212","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:63214","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":48,"response_code":"200"} {"msg":"Started propolis in zone: oxz_propolis-server_b0157dea-ec31-4457-96e1-5fc35491c963","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:30.846860161Z","hostname":"sock","pid":1574,"instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","propolis_id":"b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","component":"InstanceManager","file":"sled-agent/src/instance.rs:1787"} {"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:30.856726844Z","hostname":"sock","pid":1574,"instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","propolis_id":"b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","component":"InstanceManager","file":"sled-agent/src/instance.rs:1796"} {"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:30.894218404Z","hostname":"sock","pid":1574,"instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","propolis_id":"b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","component":"InstanceManager","file":"sled-agent/src/instance.rs:1820"} {"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:31.034358319Z","hostname":"sock","pid":1574,"instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","propolis_id":"b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","component":"InstanceManager","file":"sled-agent/src/instance.rs:932"} {"msg":"IGW mapping changed; external IPs refreshed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:31.035217605Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1212"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:31.035266406Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"7d7f419a-5fdb-4ecf-abef-cd783d2d1a9a","remote_addr":"[fd00:1122:3344:101::a]:60911","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":7482638,"response_code":"204"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Creating), migration_in: None, migration_out: None, time: 2024-11-18T21:12:31.036039672Z }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:31.036063872Z","hostname":"sock","pid":1574,"instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","propolis_id":"b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","component":"InstanceManager","file":"sled-agent/src/instance.rs:780"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:31.036082782Z","hostname":"sock","pid":1574,"instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","propolis_id":"b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","component":"InstanceManager","file":"sled-agent/src/instance.rs:802","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(3), time_updated: 2024-11-18T21:12:31.036039672Z }","propolis_id":"b0157dea-ec31-4457-96e1-5fc35491c963"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:31.036107922Z","hostname":"sock","pid":1574,"instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","propolis_id":"b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","component":"InstanceManager","file":"sled-agent/src/instance.rs:700","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Starting, gen: Generation(3), time_updated: 2024-11-18T21:12:31.036039672Z }, migration_in: None, migration_out: None }","propolis_id":"b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_in: None, migration_out: None, time: 2024-11-18T21:12:31.109256498Z }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:31.109289108Z","hostname":"sock","pid":1574,"instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","propolis_id":"b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","component":"InstanceManager","file":"sled-agent/src/instance.rs:780"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:31.109318348Z","hostname":"sock","pid":1574,"instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","propolis_id":"b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","component":"InstanceManager","file":"sled-agent/src/instance.rs:802","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(4), time_updated: 2024-11-18T21:12:31.109256498Z }","propolis_id":"b0157dea-ec31-4457-96e1-5fc35491c963"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:31.109326628Z","hostname":"sock","pid":1574,"instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","propolis_id":"b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","component":"InstanceManager","file":"sled-agent/src/instance.rs:700","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Running, gen: Generation(4), time_updated: 2024-11-18T21:12:31.109256498Z }, migration_in: None, migration_out: None }","propolis_id":"b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:31.112850155Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"b1587a1b-eb36-4b6e-a0c2-1f8ecac37b3a","remote_addr":"[fd00:1122:3344:101::a]:60911","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":349,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:35.587684918Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:57025"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:35.587818119Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"50d0ac35-92b1-4c66-b2b7-ab5ef4419e68","remote_addr":"[fd00:1122:3344:101::a]:57025","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":60,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:35.601548701Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {22b961b8-9080-4e1f-8bc7-6427df717a9f: {10.151.2.176: {3ac43a21-5b43-44f3-aa9c-4f7a4e11405c}, 10.151.2.175: {3ac43a21-5b43-44f3-aa9c-4f7a4e11405c}}} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:35.601586171Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"1a5896cf-6162-4f1f-bc03-0642fa562813","remote_addr":"[fd00:1122:3344:101::a]:57025","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":82,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:35.631426384Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:36129"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:35.633322788Z","hostname":"sock","pid":1574,"uri":"/vmms/b0157dea-ec31-4457-96e1-5fc35491c963/state","method":"GET","req_id":"14e248ee-a118-4b1b-bcae-67aacf35dbe0","remote_addr":"[fd00:1122:3344:101::a]:36129","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":990,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:35.672269748Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:52590"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:35.672567161Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"9de132cc-fb83-4917-be94-d57a80d43627","remote_addr":"[fd00:1122:3344:101::a]:52590","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":197,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:35.681972931Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"b580dba9-a0a0-4874-94c3-67433a979b46","remote_addr":"[fd00:1122:3344:101::a]:57025","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":200,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:35.769769226Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:61974"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:35.769867606Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:35.811443746Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:35.873089066Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"818317a1-4b2b-485b-919e-0e500deaa6a0","remote_addr":"[fd00:1122:3344:101::a]:61974","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":103265,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:35.873376978Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"2fd971a0-5f4d-403d-bb15-4874a576ac73","remote_addr":"[fd00:1122:3344:101::a]:61974","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":112,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:36.321812123Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:37929"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:36.321903384Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"589aae26-564c-446f-a897-8c711120a19a","remote_addr":"[fd00:1122:3344:101::b]:37929","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":38,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:36.364452711Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:33807"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:36.378666307Z","hostname":"sock","pid":1574,"uri":"/vmms/b0157dea-ec31-4457-96e1-5fc35491c963/state","method":"GET","req_id":"73a66f2a-80e2-4775-865c-426b6e87f948","remote_addr":"[fd00:1122:3344:101::c]:33807","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":144,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:36.445218433Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:38234"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:36.449889888Z","hostname":"sock","pid":1574,"uri":"/vmms/b0157dea-ec31-4457-96e1-5fc35491c963/state","method":"GET","req_id":"c1709191-a978-4a48-b594-a4bae0aed267","remote_addr":"[fd00:1122:3344:101::b]:38234","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":112,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:36.493149921Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {22b961b8-9080-4e1f-8bc7-6427df717a9f: {10.151.2.175: {3ac43a21-5b43-44f3-aa9c-4f7a4e11405c}, 10.151.2.176: {3ac43a21-5b43-44f3-aa9c-4f7a4e11405c}}} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:36.493189311Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"173e07ca-4a34-49d7-a112-29b105d882f4","remote_addr":"[fd00:1122:3344:101::b]:37929","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":125,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:36.516398304Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"9149b510-9521-4d2f-9eab-61e6bc54d9a8","remote_addr":"[fd00:1122:3344:101::b]:37929","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":237,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:36.525342301Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:59105"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:36.525476842Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"82cc624b-2b2d-406d-ae5c-bb4e1bd5fdbf","remote_addr":"[fd00:1122:3344:101::c]:59105","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":37,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:36.55609551Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:42356"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:36.556196261Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:36.60299576Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:36.666449023Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"df76125b-88d8-4eb0-9c2e-de44c79ac01c","remote_addr":"[fd00:1122:3344:101::b]:42356","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":110297,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:36.666671745Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:50605"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:36.666736755Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:38861"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:36.666814116Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::b]:61503"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:36.667042788Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"0397cf94-0070-4109-8dbb-d895801637dc","remote_addr":"[fd00:1122:3344:101::b]:42356","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":304,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:36.667071618Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {22b961b8-9080-4e1f-8bc7-6427df717a9f: {10.151.2.176: {3ac43a21-5b43-44f3-aa9c-4f7a4e11405c}, 10.151.2.175: {3ac43a21-5b43-44f3-aa9c-4f7a4e11405c}}} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:36.667163728Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"4a940304-69d4-41cc-bbe8-0198cab2d7fa","remote_addr":"[fd00:1122:3344:101::c]:59105","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":349,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:36.667262919Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"6958275d-44ec-484a-89e6-219b5e0e3866","remote_addr":"[fd00:1122:3344:101::b]:61503","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":295,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:36.66738308Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"a8ce3ab0-f4ad-48e9-9e76-c8eadccd4ffc","remote_addr":"[fd00:1122:3344:101::c]:50605","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":318,"response_code":"200"} {"msg":"ensuring scrimlet uplinks","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:36.669386915Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4172"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:36.69019417Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"6cc72d73-ca7d-4266-8bbd-0da949759bdc","remote_addr":"[fd00:1122:3344:101::c]:59105","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":5551,"response_code":"204"} {"msg":"setting up port_qsfp0","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:36.710590352Z","hostname":"sock","pid":1574,"component":"ServiceManager","file":"sled-agent/src/services.rs:4196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:36.769507151Z","hostname":"sock","pid":1574,"uri":"/switch-ports","method":"POST","req_id":"ff43fcfc-1b61-4944-95ee-dc632905e829","remote_addr":"[fd00:1122:3344:101::c]:38861","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":101040,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:36.769878194Z","hostname":"sock","pid":1574,"uri":"/network-bootstore-config","method":"GET","req_id":"468aede3-08ac-4444-aea7-989d76ba7f21","remote_addr":"[fd00:1122:3344:101::c]:38861","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":148,"response_code":"200"} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:39.967562432Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:39.968413629Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:40.003024197Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:40.003993134Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:40.092317183Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:40.140529843Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:40.14159387Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:40.142289365Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:40.14290175Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:40.143586635Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:40.14416869Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:40.144992226Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:40.167774706Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:40.1683391Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:40.169014415Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:40.169541829Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:40.170056893Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"sample queue is too full, dropping oldest samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:40.172639612Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:688","n_overflow_samples":6,"n_current_samples":500,"n_new_samples":6} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:40.92064885Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:42986"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:40.921278515Z","hostname":"sock","pid":1574,"uri":"/vmms/b0157dea-ec31-4457-96e1-5fc35491c963/state","method":"PUT","req_id":"b311f541-0b3d-4ad4-8bb6-46a1e0122b90","remote_addr":"[fd00:1122:3344:101::a]:42986","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":542,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_in: None, migration_out: None, time: 2024-11-18T21:12:40.921402526Z }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:40.921419316Z","hostname":"sock","pid":1574,"instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","propolis_id":"b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","component":"InstanceManager","file":"sled-agent/src/instance.rs:780"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:40.921437176Z","hostname":"sock","pid":1574,"instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","propolis_id":"b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","component":"InstanceManager","file":"sled-agent/src/instance.rs:802","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(6), time_updated: 2024-11-18T21:12:40.921402526Z }","propolis_id":"b0157dea-ec31-4457-96e1-5fc35491c963"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:40.921447626Z","hostname":"sock","pid":1574,"instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","propolis_id":"b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","component":"InstanceManager","file":"sled-agent/src/instance.rs:700","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(6), time_updated: 2024-11-18T21:12:40.921402526Z }, migration_in: None, migration_out: None }","propolis_id":"b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_in: None, migration_out: None, time: 2024-11-18T21:12:40.970998126Z }","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:40.971020686Z","hostname":"sock","pid":1574,"instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","propolis_id":"b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","component":"InstanceManager","file":"sled-agent/src/instance.rs:780"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:40.971056326Z","hostname":"sock","pid":1574,"instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","propolis_id":"b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","component":"InstanceManager","file":"sled-agent/src/instance.rs:802","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-11-18T21:12:40.970998126Z }","propolis_id":"b0157dea-ec31-4457-96e1-5fc35491c963"} {"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:40.971082446Z","hostname":"sock","pid":1574,"instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","propolis_id":"b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","component":"InstanceManager","file":"sled-agent/src/instance.rs:816","propolis_id":"b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:40.971113887Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"target removed with queued samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:40.971132277Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:602","n_samples":6,"id":"2945138779758151898"} {"msg":"target removed with queued samples","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:40.971180727Z","hostname":"sock","pid":1574,"component":"kstat-sampler-worker","sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"oximeter/instruments/src/kstat/sampler.rs:602","n_samples":6,"id":"17510875835644556114"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:40.971305648Z","hostname":"sock","pid":1574,"component":"StorageManager","file":"sled-storage/src/manager.rs:603"} {"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:40.971335218Z","hostname":"sock","pid":1574,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:339","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/32b20b70-d4a3-45a0-a7b8-ec67799d407c/debug/bundle/zone\", \"/pool/int/b7347e5b-0882-41c4-bf1e-7a5fbf6725b3/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/473f0b4e-a714-496d-8d5d-665782986893/crypt/debug\", \"/pool/ext/3dfe90a9-3cf2-497b-ae3f-53af8f4f1c63/crypt/debug\", \"/pool/ext/bc7fdd4e-7098-42ff-9a2a-fcfeeae662c6/crypt/debug\", \"/pool/ext/2536a198-fc1d-4175-aab5-a62f1506f3e4/crypt/debug\", \"/pool/ext/dd78d2d6-0d80-45fe-9f73-e75ef7da86ee/crypt/debug\", \"/pool/ext/28b058a3-a4b5-473e-8580-f5c606b6d30b/crypt/debug\", \"/pool/ext/80f7cc50-8d94-4fb0-b296-446e3acf3366/crypt/debug\", \"/pool/ext/feac0a3f-e4dc-4a16-9eeb-839f74ad9665/crypt/debug\", \"/pool/ext/f334d74e-134c-4206-a5ba-9a6ac6ea7954/crypt/debug\"] }","zone_name":"oxz_propolis-server_b0157dea-ec31-4457-96e1-5fc35491c963"} {"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:41.651059988Z","hostname":"sock","pid":1574,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1142","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_b0157dea-ec31-4457-96e1-5fc35491c963\", bundle_id: 854a86f6-022e-4da0-8903-f5c7b86ea19e }, time_created: 2024-11-18T21:12:40.971795372Z, version: 0, cause: TerminatedInstance }"} {"msg":"Halting and removing zone: oxz_propolis-server_b0157dea-ec31-4457-96e1-5fc35491c963","v":0,"name":"SledAgent","level":40,"time":"2024-11-18T21:12:41.651125078Z","hostname":"sock","pid":1574,"instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","propolis_id":"b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","component":"InstanceManager","file":"sled-agent/src/instance.rs:1032"} {"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:43.417185069Z","hostname":"sock","pid":1574,"instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","propolis_id":"b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","component":"InstanceManager","file":"illumos-utils/src/zone.rs:279"} {"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:43.434801261Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:964","id":"RouterId { vni: Vni(15890920), kind: System }"} {"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:43.434842261Z","hostname":"sock","pid":1574,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:964","id":"RouterId { vni: Vni(15890920), kind: Custom(V4(Ipv4Net { addr: 172.30.0.0, width: 22 })) }"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:43.435201154Z","hostname":"sock","pid":1574,"instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","propolis_id":"b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","component":"InstanceManager","file":"sled-agent/src/instance.rs:700","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(8), time_updated: 2024-11-18T21:12:43.435169854Z }, migration_in: None, migration_out: None }","propolis_id":"b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:43.474377306Z","hostname":"sock","pid":1574,"instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","propolis_id":"b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","component":"InstanceManager","file":"sled-agent/src/instance.rs:700","state":"SledVmmState { vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(8), time_updated: 2024-11-18T21:12:43.435169854Z }, migration_in: None, migration_out: None }","propolis_id":"b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d"} {"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:43.477710201Z","hostname":"sock","pid":1574,"instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","propolis_id":"b0157dea-ec31-4457-96e1-5fc35491c963","instance_id":"229c467e-2b02-41ee-af58-c382cbfa394d","component":"InstanceManager","file":"sled-agent/src/instance.rs:969"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:43.94423515Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:59430"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:43.944389421Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"c594d77a-e4fb-4ebe-a8d0-9f9d1d7494b5","remote_addr":"[fd00:1122:3344:101::c]:59430","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":85,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:44.036045595Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"IGW mapping changed; external IPs refreshed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:44.036110945Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1212"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:44.036189776Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"401c3ad1-497d-46ce-b657-94c8d8592e48","remote_addr":"[fd00:1122:3344:101::c]:59430","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":223,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:44.044307637Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"a315b134-54a7-4f59-b72d-8c0f859c13d0","remote_addr":"[fd00:1122:3344:101::c]:59430","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":180,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:44.053782877Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::c]:45313"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:44.053952778Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"b464e841-e8d9-4c96-94e4-c1111b435610","remote_addr":"[fd00:1122:3344:101::c]:45313","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":126,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:45.279029625Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:44480"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:45.279255017Z","hostname":"sock","pid":1574,"uri":"/v2p","method":"GET","req_id":"67744ca2-9815-42c6-bfc2-a805e7925f80","remote_addr":"[fd00:1122:3344:101::a]:44480","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":144,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:45.437674468Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:33021"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:45.437794669Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"065cea2d-0d6b-456c-b67f-bc7996b44028","remote_addr":"[fd00:1122:3344:101::a]:33021","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":43,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:45.452447578Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:45.452463239Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"70204326-549e-486d-9452-a03aefdcb06e","remote_addr":"[fd00:1122:3344:101::a]:33021","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":48,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:45.4594448Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"89eb24f4-c772-4d32-b271-af421080a0c8","remote_addr":"[fd00:1122:3344:101::a]:33021","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":165,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:46.188519718Z","hostname":"sock","pid":1574,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:1023","remote_addr":"[fd00:1122:3344:101::a]:59951"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:46.188671209Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"GET","req_id":"63a1d5ff-7f94-416b-9072-9857b21d997c","remote_addr":"[fd00:1122:3344:101::a]:59951","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":59,"response_code":"200"} {"msg":"IGW mapping received","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:46.202697353Z","hostname":"sock","pid":1574,"sled_id":"79cbda5a-2e27-489e-945d-860d86dec819","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1186","values":"ExternalIpGatewayMap { mappings: {} }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:46.202717774Z","hostname":"sock","pid":1574,"uri":"/eip-gateways","method":"PUT","req_id":"9b6b0e38-09f3-4f37-9411-0ce69e1e87ea","remote_addr":"[fd00:1122:3344:101::a]:59951","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":59,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-11-18T21:12:46.210358131Z","hostname":"sock","pid":1574,"uri":"/vpc-routes","method":"PUT","req_id":"ee72a369-1232-404e-a2da-44e23b7cb341","remote_addr":"[fd00:1122:3344:101::a]:59951","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/registry/src/index.crates.io-6f17d22bba15001f/dropshot-0.13.0/src/server.rs:863","latency_us":177,"response_code":"204"}